Merge remote-tracking branch 'upstream/master'

cabal.project

@@ -11,7 +11,7 @@ repository cardano-haskell-packages
 
 index-state:
   , hackage.haskell.org 2023-09-01T22:19:16Z
-  , cardano-haskell-packages 2023-10-05T21:00:00Z
+  , cardano-haskell-packages 2023-10-31T17:10:09Z
 
 packages:
   cardano-db
@@ -71,8 +71,8 @@ package snap-server
 source-repository-package
   type: git
   location: https://github.com/input-output-hk/cardano-node
-  tag: f1ce770834bf7150ca29cb647065c9e62d39be1a
-  --sha256: sha256-ONCnN1fLtYJB9kXDlUbF6nIjTnlqvI7kfppftrOOWAY=
+  tag: 735e6c97740c8def5d6e462bf5a29cbe6d323cfe
+  --sha256: sha256-Q4lhD+m5nvNu+dIHWB8WYVsWqi4/wmPPm+O/W6+HFJo=
   subdir:
     cardano-git-rev
     cardano-node

cardano-chain-gen/src/Cardano/Mock/Forging/Interpreter.hs

@@ -403,7 +403,7 @@ tryAllForging interpreter interState currentSlot xs = do
 
       -- We require the ticked ledger view in order to construct the ticked 'ChainDepState'.
       ledgerView <- case runExcept (forecastFor (istForecast interState) currentSlot) of
-        Right lv -> pure (lv :: Ticked (LedgerView (BlockProtocol CardanoBlock)))
+        Right lv -> pure (lv :: (LedgerView (BlockProtocol CardanoBlock)))
         -- Left can only happen if we cross an epoch boundary
         Left err -> throwIO $ ForecastError currentSlot err
 

cardano-db-sync/CHANGELOG.md

@@ -1,6 +1,58 @@
 # Revision history for cardano-db-sync
 
-## 13.2.0.0
+## Sancho changelog
+`sancho-a-b-c` tags in db-sync are a series of incremental integration of conway feautures. These tags can follow the
+sanchonet https://book.world.dev.cardano.org/environments.html#sanchonet-testnet.
+Their enumeration has the following meaning:<br>
+a: schema changes<br>
+b: compatible with a respinned sanchonet or node<br>
+c: minor changes or fixes<br>
+
+It is advised to resync from genesis when using a new version a or b.
+
+Some useful links (adjust the numbers to the correct tag):
+
+- Changelog: https://github.com/input-output-hk/cardano-db-sync/blob/sancho-2-0-0/cardano-db-sync/CHANGELOG.md#13200
+- Schema docs https://github.com/input-output-hk/cardano-db-sync/blob/sancho-2-0-0/doc/schema.md
+- Postgres migrations https://github.com/input-output-hk/cardano-db-sync/tree/sancho-2-0-0/schema
+
+In the schema docs, you can search for `13.2` or `Conway` for schema changes from the previous official release.
+
+### sancho-2.2.0
+- is compatible with node 8.6-pre
+- `governance_action.ratified_epoch` is now populated
+- `drep_distr.active_until` is now populated
+- Fixed typo `voting_procedure.comittee_voter` to `voting_procedure.committee_voter`
+- Added `committee_de_registration.voting_anchor_id` field
+
+### sancho-2.1.0
+- Fixed an issue with the size of Committee hash
+
+### sancho-2.0.1
+- Workaround an issue where delegation could target unregistered pools
+
+### sancho-2.0.0
+- is compatible with node 8.5-pre
+- adds the governance epoch param group in table `epoch_param` and their updates through governance proposal in
+`param_proposal`
+- adds a new field `governance_action.expiration` for the expected expiration of an action.
+- populates the `drep_distr` table and `governance.enacted_epoch` that used to remain empty.
+- fixes an issue with duplicated entries in `drep_hash` and `voting_anchor`.
+- merges drep registration, re-registration and de-registration on the same table `drep_registration` for easy use
+- the new field `drep_distr.active_until` is not populated yet.
+- change `new_committee` gov actions represantation.
+
+### sancho-1.1.0
+- is compatible with node-8.3-pre. There are no schema changes over sancho-1-0-0.
+
+### sancho-1.0.0
+The schema is quite close to the [initial design](https://github.com/input-output-hk/cardano-db-sync/blob/conway-schema-design-13.2/doc/schema.md). You may find some differences: ie the `param_proposals` and `epoch_param` are not extended yet,
+Some tables/fields are created but are not populated yet: `anchor_offline_data`, `anchor_offline_fetch_error` , `drep_distr` , `governance_action.x_epoch` , `delegation_vote.redeemer_id`
+
+### sancho-0.0.0
+It's very early stage and is missing all Conway specific feautures and some Shelley features related to Rewards. Incremental tags will follow.
+
+## 13.2.0.0 - Unreleased
 * Uses the cache for the computation of `epoch` table when following
 * `epoch_stake` is now computed earlier, before the epoch is reached
 * Exception handling and concurrency improved.

cardano-db-sync/src/Cardano/DbSync/Era/Shelley/Generic/EpochUpdate.hs

@@ -10,15 +10,14 @@ import Cardano.DbSync.Era.Shelley.Generic.ProtoParams
 import Cardano.DbSync.Types
 import Cardano.DbSync.Util
 import qualified Cardano.Ledger.BaseTypes as Ledger
-import Cardano.Ledger.Conway.Governance (EnactState)
-import Cardano.Ledger.DRepDistr
+import Cardano.Ledger.Conway.Governance
 import qualified Cardano.Ledger.Shelley.API.Wallet as Shelley
 import Cardano.Prelude hiding (Maybe (..), fromMaybe)
 import qualified Cardano.Protocol.TPraos.API as Shelley
 import qualified Cardano.Protocol.TPraos.Rules.Tickn as Shelley
 import Cardano.Slotting.Slot (EpochNo (..))
 import Data.Strict.Maybe (Maybe (..))
-import Ouroboros.Consensus.Cardano.Block (HardForkState (..), StandardConway, StandardCrypto)
+import Ouroboros.Consensus.Cardano.Block (HardForkState (..), StandardConway)
 import Ouroboros.Consensus.Cardano.CanHardFork ()
 import qualified Ouroboros.Consensus.HeaderValidation as Consensus
 import Ouroboros.Consensus.Ledger.Extended (ExtLedgerState (..))
@@ -29,7 +28,7 @@ data NewEpoch = NewEpoch
   { neEpoch :: !EpochNo
   , neIsEBB :: !Bool
   , neAdaPots :: !(Maybe Shelley.AdaPots)
-  , neDRepDistr :: !(Maybe (DRepDistr StandardCrypto))
+  , neDRepState :: !(Maybe (DRepPulsingState StandardConway))
   , neEnacted :: !(Maybe (EnactState StandardConway))
   , neEpochUpdate :: !EpochUpdate
   }

cardano-db-sync/src/Cardano/DbSync/Era/Shelley/Generic/ParamProposal.hs

@@ -87,7 +87,7 @@ convertParamProposal witness (Shelley.Update pp epoch) =
 
 -- -------------------------------------------------------------------------------------------------
 
-shelleyParamProposal :: (EraPParams era, ProtVerAtMost era 4, ProtVerAtMost era 6) => EpochNo -> Shelley.ProposedPPUpdates era -> [ParamProposal]
+shelleyParamProposal :: (EraPParams era, ProtVerAtMost era 4, ProtVerAtMost era 6, ProtVerAtMost era 8) => EpochNo -> Shelley.ProposedPPUpdates era -> [ParamProposal]
 shelleyParamProposal epochNo (Shelley.ProposedPPUpdates umap) =
   map (convertShelleyParamProposal epochNo) $ Map.toList umap
 
@@ -120,7 +120,7 @@ convertConwayParamProposal pmap =
     , pppTreasuryGrowthRate = strictMaybeToMaybe (pmap ^. ppuTauL)
     , pppDecentralisation = Nothing -- Removed in Babbage
     , pppEntropy = Nothing -- Removed in Babbage
-    , pppProtocolVersion = strictMaybeToMaybe (pmap ^. ppuProtocolVersionL)
+    , pppProtocolVersion = Nothing -- Removed in Conway
     , pppMinUtxoValue = Nothing -- Removed in Alonzo
     , pppMinPoolCost = strictMaybeToMaybe (pmap ^. ppuMinPoolCostL)
     , pppCoinsPerUtxo = unCoinPerByte <$> strictMaybeToMaybe (pmap ^. ppuCoinsPerUTxOByteL)
@@ -233,7 +233,7 @@ convertAlonzoParamProposal epochNo (key, pmap) =
     }
 
 -- | This works fine from Shelley to Mary. Not for Alonzo since 'ppuMinUTxOValueL' was removed
-convertShelleyParamProposal :: (EraPParams era, ProtVerAtMost era 4, ProtVerAtMost era 6) => EpochNo -> (Ledger.KeyHash genesis crypto, PParamsUpdate era) -> ParamProposal
+convertShelleyParamProposal :: (EraPParams era, ProtVerAtMost era 4, ProtVerAtMost era 6, ProtVerAtMost era 8) => EpochNo -> (Ledger.KeyHash genesis crypto, PParamsUpdate era) -> ParamProposal
 convertShelleyParamProposal epochNo (key, pmap) =
   ParamProposal
     { pppEpochNo = Just epochNo

cardano-db-sync/src/Cardano/DbSync/Era/Shelley/Insert.hs

@@ -69,9 +69,8 @@ import Cardano.Ledger.Compactible (fromCompact)
 import Cardano.Ledger.Conway.Core (DRepVotingThresholds (..), PoolVotingThresholds (..))
 import Cardano.Ledger.Conway.Governance
 import Cardano.Ledger.Conway.TxCert
-import Cardano.Ledger.Core
 import qualified Cardano.Ledger.Credential as Ledger
-import Cardano.Ledger.DRepDistr (extractDRepDistr)
+import Cardano.Ledger.DRep
 import Cardano.Ledger.Keys
 import qualified Cardano.Ledger.Keys as Ledger
 import Cardano.Ledger.Mary.Value (AssetName (..), MultiAsset (..), PolicyID (..))
@@ -247,10 +246,12 @@ insertOnNewEpoch tracer blkId slotNo epochNo newEpoch = do
     lift $ insertEpochParam tracer blkId epochNo params (Generic.euNonce epochUpdate)
   whenStrictJust (Generic.neAdaPots newEpoch) $ \pots ->
     insertPots blkId slotNo epochNo pots
-  whenStrictJust (Generic.neDRepDistr newEpoch) $ \dreps ->
-    lift $ insertDrepDistr epochNo $ extractDRepDistr dreps
+  whenStrictJust (Generic.neDRepState newEpoch) $ \dreps -> do
+    let (drepSnapshot, ratifyState) = finishDRepPulser dreps
+    lift $ insertDrepDistr epochNo drepSnapshot
+    updateEnacted False epochNo (rsEnactState ratifyState)
   whenStrictJust (Generic.neEnacted newEpoch) $ \enactedSt ->
-    updateEnacted epochNo enactedSt
+    updateEnacted True epochNo enactedSt
   where
     epochUpdate :: Generic.EpochUpdate
     epochUpdate = Generic.neEpochUpdate newEpoch
@@ -525,8 +526,8 @@ insertCertificate tracer cache isMember network blkId txId epochNo slotNo redeem
         lift $ insertDrepDeRegistration txId idx cred coin
       ConwayAuthCommitteeHotKey khCold khHot ->
         lift $ insertCommitteeRegistration txId idx khCold khHot
-      ConwayResignCommitteeColdKey khCold ->
-        lift $ insertCommitteeDeRegistration txId idx khCold
+      ConwayResignCommitteeColdKey khCold anchor ->
+        lift $ insertCommitteeDeRegistration txId idx khCold (strictMaybeToMaybe anchor)
       ConwayUpdateDRep cred anchor ->
         lift $ insertDrepRegistration txId idx cred Nothing (strictMaybeToMaybe anchor)
   where
@@ -553,13 +554,16 @@ insertCommitteeDeRegistration ::
   DB.TxId ->
   Word16 ->
   Ledger.Credential 'ColdCommitteeRole StandardCrypto ->
+  Maybe (Anchor StandardCrypto) ->
   ReaderT SqlBackend m ()
-insertCommitteeDeRegistration txId idx khCold = do
+insertCommitteeDeRegistration txId idx khCold mAnchor = do
+  votingAnchorId <- whenMaybe mAnchor $ insertAnchor txId
   void . DB.insertCommitteeDeRegistration $
     DB.CommitteeDeRegistration
       { DB.committeeDeRegistrationTxId = txId
       , DB.committeeDeRegistrationCertIndex = idx
       , DB.committeeDeRegistrationHotKey = Generic.unCredentialHash khCold
+      , DB.committeeDeRegistrationVotingAnchorId = votingAnchorId
       }
 
 insertDrepRegistration ::
@@ -1531,7 +1535,7 @@ insertVotingProcedure ::
 insertVotingProcedure cache txId voter (index, (gaId, vp)) = do
   govActionId <- resolveGovernanceAction gaId
   votingAnchorId <- whenMaybe (strictMaybeToMaybe $ vProcAnchor vp) $ lift . insertAnchor txId
-  (mComitteeVoter, mDRepVoter, mStakePoolVoter) <- case voter of
+  (mCommitteeVoter, mDRepVoter, mStakePoolVoter) <- case voter of
     CommitteeVoter cred ->
       pure (Just $ Generic.unCredentialHash cred, Nothing, Nothing)
     DRepVoter cred -> do
@@ -1545,7 +1549,7 @@ insertVotingProcedure cache txId voter (index, (gaId, vp)) = do
       { DB.votingProcedureTxId = txId
       , DB.votingProcedureIndex = index
       , DB.votingProcedureGovernanceActionId = govActionId
-      , DB.votingProcedureComitteeVoter = mComitteeVoter
+      , DB.votingProcedureCommitteeVoter = mCommitteeVoter
       , DB.votingProcedureDrepVoter = mDRepVoter
       , DB.votingProcedurePoolVoter = mStakePoolVoter
       , DB.votingProcedureVoterRole = Generic.toVoterRole voter
@@ -1596,9 +1600,9 @@ insertCredDrepHash cred = do
   where
     bs = Generic.unCredentialHash cred
 
-insertDrepDistr :: forall m. (MonadBaseControl IO m, MonadIO m) => EpochNo -> Map (DRep StandardCrypto) (Ledger.CompactForm Coin) -> ReaderT SqlBackend m ()
-insertDrepDistr e dreps = do
-  drepsDB <- mapM mkEntry (Map.toList dreps)
+insertDrepDistr :: forall m. (MonadBaseControl IO m, MonadIO m) => EpochNo -> PulsingSnapshot StandardConway -> ReaderT SqlBackend m ()
+insertDrepDistr e pSnapshot = do
+  drepsDB <- mapM mkEntry (Map.toList $ psDRepDistr pSnapshot)
   DB.insertManyDrepDistr drepsDB
   where
     mkEntry :: (DRep StandardCrypto, Ledger.CompactForm Coin) -> ReaderT SqlBackend m DB.DrepDistr
@@ -1609,13 +1613,21 @@ insertDrepDistr e dreps = do
           { DB.drepDistrHashId = drepId
           , DB.drepDistrAmount = fromIntegral $ unCoin $ fromCompact coin
           , DB.drepDistrEpochNo = unEpochNo e
-          , DB.drepDistrActiveUntil = Nothing
+          , DB.drepDistrActiveUntil = unEpochNo <$> isActiveEpochNo drep
           }
 
-updateEnacted :: forall m. (MonadBaseControl IO m, MonadIO m) => EpochNo -> EnactState StandardConway -> ExceptT SyncNodeError (ReaderT SqlBackend m) ()
-updateEnacted epochNo enactedState = do
+    isActiveEpochNo :: DRep StandardCrypto -> Maybe EpochNo
+    isActiveEpochNo = \case
+      DRepAlwaysAbstain -> Nothing
+      DRepAlwaysNoConfidence -> Nothing
+      DRepCredential cred -> drepExpiry <$> Map.lookup cred (psDRepState pSnapshot)
+
+updateEnacted :: forall m. (MonadBaseControl IO m, MonadIO m) => Bool -> EpochNo -> EnactState StandardConway -> ExceptT SyncNodeError (ReaderT SqlBackend m) ()
+updateEnacted isEnacted epochNo enactedState = do
   whenJust (strictMaybeToMaybe (enactedState ^. ensPrevPParamUpdateL)) $ \prevId -> do
     gaId <- resolveGovernanceAction $ getPrevId prevId
-    lift $ DB.updateGovAction gaId (unEpochNo epochNo)
+    if isEnacted
+      then lift $ DB.updateGovActionEnacted gaId (unEpochNo epochNo)
+      else lift $ DB.updateGovActionRatified gaId (unEpochNo epochNo)
   where
     getPrevId (PrevGovActionId gai) = gai

cardano-db-sync/src/Cardano/DbSync/Ledger/State.hs

@@ -76,7 +76,6 @@ import Cardano.Ledger.Conway.Core as Shelley
 import Cardano.Ledger.Conway.Governance
 import qualified Cardano.Ledger.Conway.Governance as Shelley
 import qualified Cardano.Ledger.Conway.PParams as Shelley
-import Cardano.Ledger.DRepDistr
 import qualified Data.ByteString.Char8 as BS
 import qualified Data.ByteString.Lazy.Char8 as LBS
 import qualified Data.ByteString.Short as SBS
@@ -264,7 +263,7 @@ applyBlock env blk = do
                     , Generic.neIsEBB = isJust $ blockIsEBB blk
                     , Generic.neAdaPots = maybeToStrict mPots
                     , Generic.neEpochUpdate = Generic.epochUpdate newState
-                    , Generic.neDRepDistr = maybeToStrict $ getDrepDistr newState
+                    , Generic.neDRepState = maybeToStrict $ getDrepDistr newState
                     , Generic.neEnacted = maybeToStrict $ getEnacted newState
                     }
 
@@ -275,10 +274,10 @@ applyBlock env blk = do
     applyToEpochBlockNo _ _ GenesisEpochBlockNo = EpochBlockNo 0
     applyToEpochBlockNo _ _ EBBEpochBlockNo = EpochBlockNo 0
 
-getDrepDistr :: ExtLedgerState CardanoBlock -> Maybe (DRepDistr StandardCrypto)
+getDrepDistr :: ExtLedgerState CardanoBlock -> Maybe (DRepPulsingState StandardConway)
 getDrepDistr ls = case ledgerState ls of
   LedgerStateConway cls ->
-    Just $ Consensus.shelleyLedgerState cls ^. Shelley.newEpochStateDRepDistrL
+    Just $ Consensus.shelleyLedgerState cls ^. Shelley.newEpochStateDRepPulsingStateL
   _ -> Nothing
 
 getEnacted :: ExtLedgerState CardanoBlock -> Maybe (EnactState StandardConway)

cardano-db-sync/src/Cardano/DbSync/Rollback.hs

@@ -53,6 +53,7 @@ rollbackFromBlockNo syncEnv blkNo = do
         DB.setNullTxOut trce (DB.minTxInId minIds) txInDeleted
       DB.deleteEpochRows epochNo
       DB.setNullEnacted epochNo
+      DB.setNullRatified epochNo
       when (deletedBlockCount > 0) $ do
         -- We use custom constraints to improve input speeds when syncing.
         -- If they don't already exists we add them here as once a rollback has happened

cardano-db/src/Cardano/Db/Insert.hs

@@ -59,8 +59,10 @@ module Cardano.Db.Insert (
   insertExtraMigration,
   insertEpochStakeProgress,
   updateSetComplete,
-  updateGovAction,
+  updateGovActionEnacted,
+  updateGovActionRatified,
   setNullEnacted,
+  setNullRatified,
   replaceAdaPots,
   insertAnchor,
   insertGovernanceAction,
@@ -330,14 +332,22 @@ updateSetComplete :: MonadIO m => Word64 -> ReaderT SqlBackend m ()
 updateSetComplete epoch = do
   updateWhere [EpochStakeProgressEpochNo Database.Persist.==. epoch] [EpochStakeProgressCompleted Database.Persist.=. True]
 
-updateGovAction :: MonadIO m => GovernanceActionId -> Word64 -> ReaderT SqlBackend m ()
-updateGovAction gaid eNo =
+updateGovActionEnacted :: MonadIO m => GovernanceActionId -> Word64 -> ReaderT SqlBackend m ()
+updateGovActionEnacted gaid eNo =
   updateWhere [GovernanceActionId ==. gaid, GovernanceActionEnactedEpoch ==. Nothing] [GovernanceActionEnactedEpoch =. Just eNo]
 
+updateGovActionRatified :: MonadIO m => GovernanceActionId -> Word64 -> ReaderT SqlBackend m ()
+updateGovActionRatified gaid eNo =
+  updateWhere [GovernanceActionId ==. gaid, GovernanceActionRatifiedEpoch ==. Nothing] [GovernanceActionRatifiedEpoch =. Just eNo]
+
 setNullEnacted :: MonadIO m => Word64 -> ReaderT SqlBackend m ()
 setNullEnacted eNo =
   updateWhere [GovernanceActionEnactedEpoch !=. Nothing, GovernanceActionEnactedEpoch >. Just eNo] [GovernanceActionEnactedEpoch =. Nothing]
 
+setNullRatified :: MonadIO m => Word64 -> ReaderT SqlBackend m ()
+setNullRatified eNo =
+  updateWhere [GovernanceActionRatifiedEpoch !=. Nothing, GovernanceActionRatifiedEpoch >. Just eNo] [GovernanceActionRatifiedEpoch =. Nothing]
+
 replaceAdaPots :: (MonadBaseControl IO m, MonadIO m) => BlockId -> AdaPots -> ReaderT SqlBackend m Bool
 replaceAdaPots blockId adapots = do
   mAdaPotsId <- queryAdaPotsId blockId

cardano-db/src/Cardano/Db/Schema.hs

@@ -564,6 +564,7 @@ share
     txId                TxId                noreference
     certIndex           Word16
     hotKey              ByteString          sqltype=bytea
+    votingAnchorId      VotingAnchorId Maybe  noreference
 
   DrepRegistration
     txId                TxId                noreference
@@ -609,7 +610,7 @@ share
     index                Word16
     governanceActionId   GovernanceActionId   noreference
     voterRole            VoterRole            sqltype=voterrole
-    comitteeVoter        ByteString Maybe
+    committeeVoter       ByteString Maybe
     drepVoter            DrepHashId Maybe     noreference
     poolVoter            PoolHashId Maybe     noreference
     vote                 Vote                 sqltype=vote
@@ -1167,6 +1168,7 @@ schemaDocs =
       CommitteeDeRegistrationTxId # "The Tx table index of the tx that includes this certificate."
       CommitteeDeRegistrationCertIndex # "The index of this deregistration within the certificates of this transaction."
       CommitteeDeRegistrationHotKey # "The deregistered hot key hash"
+      CommitteeDeRegistrationVotingAnchorId # "The Voting anchor reference id"
 
     DrepRegistration --^ do
       "A table for DRep registrations, deregistrations or updates. Registration have positive deposit values, deregistrations have negative and\
@@ -1223,6 +1225,7 @@ schemaDocs =
       VotingProcedureIndex # "The index of this VotingProcedure within this transaction."
       VotingProcedureGovernanceActionId # "The index of the GovernanceAction that this vote targets."
       VotingProcedureVoterRole # "The role of the voter. Can be one of ConstitutionalCommittee, DRep, SPO."
+      VotingProcedureCommitteeVoter # ""
       VotingProcedureVote # "The Vote. Can be one of Yes, No, Abstain."
       VotingProcedureVotingAnchorId # "The VotingAnchor table index associated with this VotingProcedure."