Merge pull request #2066 from nervosnetwork/testnet

Deploy to mainnet

app/controllers/api/v1/udts_controller.rb

@@ -93,6 +93,7 @@ def holder_allocation
           {
             name: allocation.contract.name,
             code_hash: allocation.contract.code_hash,
+            hash_type: allocation.contract.hash_type,
             holder_count: allocation.ckb_holder_count,
           }
         end

app/models/cell_output.rb

@@ -1,6 +1,8 @@
 class CellOutput < ApplicationRecord
   include CellOutputs::ExtraInfo
 
+  self.primary_key = :id
+
   SYSTEM_TX_HASH = "0x0000000000000000000000000000000000000000000000000000000000000000".freeze
   MAXIMUM_DOWNLOADABLE_SIZE = 64000
   MIN_SUDT_AMOUNT_BYTESIZE = 16
@@ -298,11 +300,10 @@ def self.update_cell_types_for_cota
 #
 #  id                       :bigint           not null, primary key
 #  capacity                 :decimal(64, 2)
-#  data                     :binary
 #  ckb_transaction_id       :bigint
 #  created_at               :datetime         not null
 #  updated_at               :datetime         not null
-#  status                   :integer          default("live")
+#  status                   :integer          default("live"), not null
 #  address_id               :decimal(30, )
 #  block_id                 :decimal(30, )
 #  tx_hash                  :binary
@@ -322,17 +323,13 @@ def self.update_cell_types_for_cota
 #
 # Indexes
 #
-#  index_cell_outputs_on_address_id_and_status              (address_id,status)
+#  index_cell_outputs_on_address_id                         (address_id)
 #  index_cell_outputs_on_block_id                           (block_id)
 #  index_cell_outputs_on_block_timestamp                    (block_timestamp)
-#  index_cell_outputs_on_cell_type                          (cell_type)
-#  index_cell_outputs_on_ckb_transaction_id_and_cell_index  (ckb_transaction_id,cell_index) UNIQUE
 #  index_cell_outputs_on_consumed_block_timestamp           (consumed_block_timestamp)
 #  index_cell_outputs_on_consumed_by_id                     (consumed_by_id)
-#  index_cell_outputs_on_data_hash                          (data_hash) USING hash
 #  index_cell_outputs_on_lock_script_id                     (lock_script_id)
-#  index_cell_outputs_on_status                             (status)
-#  index_cell_outputs_on_tx_hash_and_cell_index             (tx_hash,cell_index) UNIQUE
+#  index_cell_outputs_on_tx_hash_and_cell_index_and_status  (tx_hash,cell_index,status) UNIQUE
+#  index_cell_outputs_on_tx_id_and_cell_index_and_status    (ckb_transaction_id,cell_index,status) UNIQUE
 #  index_cell_outputs_on_type_script_id                     (type_script_id)
-#  index_cell_outputs_on_type_script_id_and_id              (type_script_id,id)
 #

app/models/ckb_sync/new_node_data_processor.rb

@@ -846,7 +846,10 @@ def build_cells_and_locks!(
       build_cell_outputs!(node_block, outputs, ckb_txs, local_block, cell_outputs_attributes, output_capacities, tags,
                           udt_address_ids, dao_address_ids, contained_udt_ids, contained_addr_ids, addrs_changes, token_transfer_ckb_tx_ids)
       if cell_outputs_attributes.present?
-        id_hashes = CellOutput.upsert_all(cell_outputs_attributes, unique_by: %i[tx_hash cell_index],
+        tx_hashes = cell_outputs_attributes.map { |attr| attr[:tx_hash] }
+        binary_hashes = CkbUtils.hexes_to_bins_sql(tx_hashes)
+        CellOutput.pending.where("tx_hash IN (#{binary_hashes})").update_all(status: :live)
+        id_hashes = CellOutput.upsert_all(cell_outputs_attributes, unique_by: %i[tx_hash cell_index status],
                                                                    returning: %i[id data_hash])
         cell_data_attrs = []
 
@@ -873,8 +876,11 @@ def build_cells_and_locks!(
       CellInput.upsert_all(cell_inputs_attributes,
                            unique_by: %i[ckb_transaction_id index])
       if prev_cell_outputs_attributes.present?
+        cell_ouput_ids = prev_cell_outputs_attributes.map { |attr| attr[:id] }
+        CellOutput.live.where(id: cell_ouput_ids).update_all(status: :dead)
         CellOutput.upsert_all(prev_cell_outputs_attributes,
-                              unique_by: %i[tx_hash cell_index])
+                              unique_by: %i[tx_hash cell_index status],
+                              record_timestamps: true)
       end
 
       ScriptTransaction.create_from_scripts TypeScript.where(id: type_script_ids)
@@ -906,13 +912,13 @@ def prepare_script_ids(outputs)
           local_cache.fetch("NodeData/LockScript/#{output.lock.code_hash}-#{output.lock.hash_type}-#{output.lock.args}") do
             # TODO use LockScript.where(script_hash: output.lock.compute_hash).select(:id)&.first replace search by code_hash, hash_type and args query after script_hash has been filled
             LockScript.where(code_hash: output.lock.code_hash, hash_type: output.lock.hash_type,
-                             args: output.lock.args).select(:id).take!
+                             args: output.lock.args).take!
           end
           if output.type.present?
             local_cache.fetch("NodeData/TypeScript/#{output.type.code_hash}-#{output.type.hash_type}-#{output.type.args}") do
               # TODO use TypeScript.where(script_hash: output.type.compute_hash).select(:id)&.first replace search by code_hash, hash_type and args query after script_hash has been filled
               TypeScript.where(code_hash: output.type.code_hash, hash_type: output.type.hash_type,
-                               args: output.type.args).select(:id).take!
+                               args: output.type.args).take!
             end
           end
         end
@@ -1141,7 +1147,7 @@ def cell_output_attributes(output, address, ckb_transaction, local_block, cell_i
       attrs = {
         ckb_transaction_id: ckb_transaction["id"],
         capacity: output.capacity,
-        occupied_capacity: 0,
+        occupied_capacity: CkbUtils.cal_cell_min_capacity(lock_script, type_script, output.capacity, binary_data),
         address_id: address.id,
         block_id: local_block.id,
         tx_hash: ckb_transaction["tx_hash"],
@@ -1221,11 +1227,9 @@ def cell_input_attributes(input, ckb_transaction_id, local_block_id,
           previous_output: {
             id: previous_output.id,
             cell_type: previous_output.cell_type,
-            created_at: previous_output.created_at,
             tx_hash: input.previous_output.tx_hash,
             cell_index: input.previous_output.index,
             status: "dead",
-            updated_at: Time.current,
             consumed_by_id: ckb_transaction_id,
             consumed_block_timestamp: CkbTransaction.find(ckb_transaction_id).block_timestamp,
           },

app/models/daily_statistic.rb

@@ -5,7 +5,7 @@ class DailyStatistic < ApplicationRecord
     transactions_count addresses_count total_dao_deposit live_cells_count dead_cells_count avg_hash_rate avg_difficulty uncle_rate
     total_depositors_count address_balance_distribution total_tx_fee occupied_capacity daily_dao_deposit daily_dao_depositors_count
     circulation_ratio daily_dao_withdraw nodes_count circulating_supply burnt locked_capacity treasury_amount mining_reward
-    deposit_compensation liquidity created_at_unixtimestamp ckb_hodl_wave
+    deposit_compensation liquidity created_at_unixtimestamp ckb_hodl_wave holder_count
   ).freeze
   MILLISECONDS_IN_DAY = BigDecimal(24 * 60 * 60 * 1000)
   GENESIS_TIMESTAMP = 1573852190812
@@ -398,6 +398,17 @@ def liquidity
     base_attrs.merge(total_supply: total_live_capacities).transform_values { |value| (value.to_f / 10**8).truncate(8) }
   end
 
+  define_logic :holder_count do
+    ActiveRecord::Base.connection.execute("SET statement_timeout = 0")
+    live_query = CellOutput.live.generated_before(to_be_counted_date.to_i * 1000 - 1).select(:address_id).to_sql
+    dead_query = CellOutput.dead.generated_before(to_be_counted_date.to_i * 1000 - 1).consumed_after(to_be_counted_date.to_i * 1000).select(:address_id).to_sql
+    combined_query = "#{live_query} UNION #{dead_query}"
+    count_query = "SELECT COUNT(DISTINCT address_id) AS count FROM (#{combined_query}) AS combined_results;"
+    count = ActiveRecord::Base.connection.execute(count_query).first["count"]
+    ActiveRecord::Base.connection.execute("RESET statement_timeout")
+    count
+  end
+
   private
 
   def to_be_counted_date
@@ -558,6 +569,7 @@ def aggron_first_day?
 #  nodes_count                  :integer
 #  locked_capacity              :decimal(30, )
 #  ckb_hodl_wave                :jsonb
+#  holder_count                 :integer
 #
 # Indexes
 #

app/models/referring_cell.rb

@@ -56,6 +56,5 @@ def self.create_initial_data_for_ckb_transaction(ckb_transaction)
 #
 # Indexes
 #
-#  index_referring_cells_on_cell_output_id                  (cell_output_id) UNIQUE
 #  index_referring_cells_on_contract_id_and_cell_output_id  (contract_id,cell_output_id) UNIQUE
 #

app/serializers/daily_statistic_serializer.rb

@@ -128,4 +128,10 @@ class DailyStatisticSerializer
   attribute :ckb_hodl_wave, if: Proc.new { |_record, params|
     params.present? && params[:indicator].include?("ckb_hodl_wave")
   }
+
+  attribute :holder_count, if: Proc.new { |_record, params|
+    params.present? && params[:indicator].include?("holder_count")
+  } do |object|
+    object.holder_count.to_s
+  end
 end

app/services/charts/daily_statistic_generator.rb

@@ -36,7 +36,7 @@ def updated_attrs
         treasury_amount estimated_apc live_cells_count dead_cells_count avg_hash_rate
         avg_difficulty uncle_rate address_balance_distribution
         total_tx_fee occupied_capacity daily_dao_deposit total_supply block_time_distribution
-        epoch_time_distribution epoch_length_distribution locked_capacity ckb_hodl_wave
+        epoch_time_distribution epoch_length_distribution locked_capacity ckb_hodl_wave holder_count
       }
 
       established_order + others

app/utils/ckb_utils.rb

@@ -11,6 +11,14 @@ def self.calculate_cell_min_capacity(output, data)
     output.calculate_min_capacity(data)
   end
 
+  def self.cal_cell_min_capacity(lock_script, type_script, capacity, binary_data)
+    lock = CKB::Types::Script.new(**lock_script.to_node)
+    type = type_script.present? ? CKB::Types::Script.new(**type_script.to_node) : nil
+    CKB::Types::Output.new(capacity: capacity.to_i, lock:, type:)
+    CKB::Utils.byte_to_shannon([8, binary_data&.bytesize || 0, lock_script.calculate_bytesize,
+                                type_script&.calculate_bytesize || 0].sum)
+  end
+
   def self.block_cell_consumed(transactions)
     transactions.reduce(0) do |memo, transaction|
       outputs_data = transaction.outputs_data

app/workers/generate_statistics_data_worker.rb

@@ -14,26 +14,5 @@ def perform(block_id)
     if epoch_stats && epoch_stats.largest_block_size.to_i < block_size
       epoch_stats.update(largest_block_size: block_size, largest_block_number: block.number)
     end
-
-    cell_outputs = block.cell_outputs.select(:id, :created_at, :data, :capacity, :lock_script_id, :type_script_id).to_a
-    cell_outputs_attributes = []
-    cell_outputs.each do |cell_output|
-      data_size =
-        if cell_output.data
-          CKB::Utils.hex_to_bin(cell_output.data).bytesize
-        else
-          0
-        end
-
-      cell_outputs_attributes << {
-        id: cell_output.id,
-        data_size:,
-        occupied_capacity: CkbUtils.calculate_cell_min_capacity(cell_output.node_output, cell_output.data),
-        created_at: cell_output.created_at,
-        updated_at: Time.current,
-      }
-    end
-
-    CellOutput.upsert_all(cell_outputs_attributes) if cell_outputs_attributes.present?
   end
 end

config/database.yml

@@ -21,14 +21,14 @@ default: &default
   # http://guides.rubyonrails.org/configuring.html#database-pooling
   host: <%= ENV.fetch("DB_HOST") { "localhost" } %>
   port: <%= ENV.fetch("DB_PORT") { "5432" } %>
-  pool: <%= ENV.fetch("RAILS_MAX_THREADS") { 25 } %>
+  pool: <%= ENV.fetch("DB_POOL")  { 5 } %>
   username: <%= ENV.fetch("DB_USERNAME") { "postgres" } %>
   password: <%= ENV.fetch("DB_PASSWORD") { "postgres" } %>
   reaping_frequency: 60 # every 60s check pool
   timeout: 5000 # over 5s will timeout
   variables:
-    statement_timeout: 120000 # query timemout over 120s
-    idle_in_transaction_session_timeout: 60000 # idle transaction over 60s
+    statement_timeout: 60000 # query timemout over 60s
+    idle_in_transaction_session_timeout: 120000 # idle transaction over 120s
 
 development:
   <<: *default
@@ -88,5 +88,5 @@ test:
 #     url: <%= ENV['DATABASE_URL'] %>
 #
 production:
+  <<: *default
   url: <%= ENV['DATABASE_URL'] %>
-  pool: <%= ENV["DB_POOL"] || ENV['RAILS_MAX_THREADS'] || 5 %>

db/migrate/20240704092919_add_holder_count_to_daily_statistic.rb

@@ -0,0 +1,5 @@
+class AddHolderCountToDailyStatistic < ActiveRecord::Migration[7.0]
+  def change
+    add_column :daily_statistics, :holder_count, :integer
+  end
+end

db/migrate/20240709131020_rename_cell_outputs_to_cell_outputs_old.rb

@@ -0,0 +1,9 @@
+class RenameCellOutputsToCellOutputsOld < ActiveRecord::Migration[7.0]
+  def up
+    rename_table :cell_outputs, :cell_outputs_old
+  end
+
+  def down
+    rename_table :cell_outputs_old, :cell_outputs
+  end
+end

db/migrate/20240709131132_create_cell_outputs.rb

@@ -0,0 +1,47 @@
+class CreateCellOutputs < ActiveRecord::Migration[7.0]
+  def up
+    execute <<-SQL
+      CREATE TABLE cell_outputs (
+        id bigserial NOT NULL,
+        capacity numeric(64,2),
+        ckb_transaction_id bigint,
+        created_at timestamp without time zone NOT NULL,
+        updated_at timestamp without time zone NOT NULL,
+        status smallint DEFAULT 0,
+        address_id numeric(30,0),
+        block_id numeric(30,0),
+        tx_hash bytea,
+        cell_index integer,
+        consumed_by_id numeric(30,0),
+        cell_type integer DEFAULT 0,
+        data_size integer,
+        occupied_capacity numeric(30,0),
+        block_timestamp numeric(30,0),
+        consumed_block_timestamp numeric(30,0),
+        type_hash character varying,
+        udt_amount numeric(40,0),
+        dao character varying,
+        lock_script_id bigint,
+        type_script_id bigint,
+        data_hash bytea,
+        primary key (id, status)
+      )  PARTITION BY LIST (status);
+
+      CREATE TABLE cell_outputs_live PARTITION OF cell_outputs
+      FOR VALUES IN (0);
+
+      CREATE TABLE cell_outputs_dead PARTITION OF cell_outputs
+      FOR VALUES IN (1);
+
+      CREATE TABLE cell_outputs_pending PARTITION OF cell_outputs
+      FOR VALUES IN (2);
+
+      CREATE TABLE cell_outputs_rejected PARTITION OF cell_outputs
+      FOR VALUES IN (3);
+    SQL
+  end
+
+  def down
+    drop_table :cell_outputs
+  end
+end

db/migrate/20240709131713_import_cell_outputs_old_to_cell_outputs.rb

@@ -0,0 +1,18 @@
+class ImportCellOutputsOldToCellOutputs < ActiveRecord::Migration[7.0]
+  def up
+    execute <<~SQL
+      SET statement_timeout = 0;
+
+      INSERT INTO cell_outputs (id, capacity, ckb_transaction_id, status, address_id, block_id, tx_hash, cell_index, consumed_by_id, cell_type, data_size, occupied_capacity, block_timestamp, consumed_block_timestamp, type_hash, udt_amount, dao, lock_script_id, type_script_id, data_hash, created_at, updated_at)
+      SELECT id, capacity, ckb_transaction_id, status, address_id, block_id, tx_hash, cell_index, consumed_by_id, cell_type, data_size, occupied_capacity, block_timestamp, consumed_block_timestamp, type_hash, udt_amount, dao, lock_script_id, type_script_id, data_hash, created_at, updated_at FROM cell_outputs_old;
+
+      SELECT setval('cell_outputs_id_seq', (SELECT max(id) FROM cell_outputs));
+    SQL
+  end
+
+  def down
+    execute <<~SQL
+      TRUNCATE TABLE cell_outputs;
+    SQL
+  end
+end

db/migrate/20240709142013_add_index_to_cell_outputs.rb

@@ -0,0 +1,13 @@
+class AddIndexToCellOutputs < ActiveRecord::Migration[7.0]
+  def change
+    add_index :cell_outputs, :address_id
+    add_index :cell_outputs, :block_id
+    add_index :cell_outputs, :consumed_by_id
+    add_index :cell_outputs, :lock_script_id
+    add_index :cell_outputs, :type_script_id
+    add_index :cell_outputs, %i[ckb_transaction_id cell_index status], unique: true, name: "index_cell_outputs_on_tx_id_and_cell_index_and_status"
+    add_index :cell_outputs, %i[tx_hash cell_index status], unique: true, name: "index_cell_outputs_on_tx_hash_and_cell_index_and_status"
+    add_index :cell_outputs, :block_timestamp
+    add_index :cell_outputs, :consumed_block_timestamp
+  end
+end