store/postgres: insist on a database with C locale

Author: saihaj

NEWS.md

@@ -8,6 +8,7 @@
   - The two affected deployments are: `Qmccst5mbV5a6vT6VvJMLPKMAA1VRgT6NGbxkLL8eDRsE7` and `Qmd9nZKCH8UZU1pBzk7G8ECJr3jX3a2vAf3vowuTwFvrQg`;
   - Here's an example [manifest](https://ipfs.io/ipfs/Qmd9nZKCH8UZU1pBzk7G8ECJr3jX3a2vAf3vowuTwFvrQg), taking a look at the data sources of name `ERC721` and `CryptoKitties`, both listen to the `Transfer(...)` event. Considering a block where there's only one occurence of this event, `graph-node` would duplicate it and call `handleTransfer` twice. Now this is fixed and it will be called only once per event/call that happened on chain.
   - In the case you're indexing one of those, you should first upgrade the `graph-node` version, then rewind the affected subgraphs to the smallest `startBlock` of their subgraph manifest. To achieve that the `graphman rewind` CLI command can be used.
+- We now check that the database uses the `C` locale and `UTF8` character encoding. For new installations, `graph-node` will panic on startup if the database uses any other locale. The easiest way to make sure this check passes is to create the database cluster with `initdb -E UTF8 --locale C`. We will provide instructions on migrating existing installations in the future.
 
 ## v0.28.2
 
@@ -42,7 +43,7 @@ Yanked. Please migrate to `v0.28.2`.
   E.g. `$ graphman --node-id index_node_0 --config graph-node.toml config provider mainnet`
 - Experimental support for GraphQL API versioning has landed. [#3185](https://github.com/graphprotocol/graph-node/pull/3185)
 - Progress towards experimental support for off-chain data sources. [#3791](https://github.com/graphprotocol/graph-node/pull/3791)
-- Experimental integration for substreams. [#3777](https://github.com/graphprotocol/graph-node/pull/3777), [#3784](https://github.com/graphprotocol/graph-node/pull/3784), [#3897](https://github.com/graphprotocol/graph-node/pull/3897), [#3765](https://github.com/graphprotocol/graph-node/pull/3765), and others 
+- Experimental integration for substreams. [#3777](https://github.com/graphprotocol/graph-node/pull/3777), [#3784](https://github.com/graphprotocol/graph-node/pull/3784), [#3897](https://github.com/graphprotocol/graph-node/pull/3897), [#3765](https://github.com/graphprotocol/graph-node/pull/3765), and others
 
 ### Bug fixes
 

store/postgres/src/catalog.rs

@@ -42,6 +42,14 @@ table! {
         nspname -> Text,
     }
 }
+// Readonly; only mapping the columns we want
+table! {
+    pg_database(datname) {
+        datname -> Text,
+        datcollate -> Text,
+        datctype -> Text,
+    }
+}
 
 table! {
     subgraphs.table_stats {
@@ -72,6 +80,56 @@ const CREATE_EXCLUSION_CONSTRAINT: bool = true;
 #[cfg(not(debug_assertions))]
 const CREATE_EXCLUSION_CONSTRAINT: bool = false;
 
+pub struct Locale {
+    collate: String,
+    ctype: String,
+    encoding: String,
+}
+
+impl Locale {
+    /// Load locale information for current database
+    pub fn load(conn: &PgConnection) -> Result<Locale, StoreError> {
+        use diesel::dsl::sql;
+        use pg_database as db;
+
+        let (collate, ctype, encoding) = db::table
+            .filter(db::datname.eq(sql("current_database()")))
+            .select((
+                db::datcollate,
+                db::datctype,
+                sql::<Text>("pg_encoding_to_char(encoding)::text"),
+            ))
+            .get_result::<(String, String, String)>(conn)?;
+        Ok(Locale {
+            collate,
+            ctype,
+            encoding,
+        })
+    }
+
+    pub fn suitable(&self) -> Result<(), String> {
+        if self.collate != "C" {
+            return Err(format!(
+                "database collation is `{}` but must be `C`",
+                self.collate
+            ));
+        }
+        if self.ctype != "C" {
+            return Err(format!(
+                "database ctype is `{}` but must be `C`",
+                self.ctype
+            ));
+        }
+        if self.encoding != "UTF8" {
+            return Err(format!(
+                "database encoding is `{}` but must be `UTF8`",
+                self.encoding
+            ));
+        }
+        Ok(())
+    }
+}
+
 /// Information about what tables and columns we have in the database
 #[derive(Debug, Clone)]
 pub struct Catalog {

store/postgres/src/connection_pool.rs

@@ -980,6 +980,19 @@ impl PoolInner {
         let conn = self.get().map_err(|_| StoreError::DatabaseUnavailable)?;
 
         let start = Instant::now();
+        if let Err(msg) = catalog::Locale::load(&conn)?.suitable() {
+            if &self.shard == &*PRIMARY_SHARD && primary::is_empty(&conn)? {
+                die(
+                    &pool.logger,
+                    "Database does not use C locale. \
+                    Please check the graph-node documentation for how to set up the database locale",
+                    &msg,
+                );
+            } else {
+                warn!(pool.logger, "{}.\nPlease check the graph-node documentation for how to set up the database locale", msg);
+            }
+        }
+
         advisory_lock::lock_migration(&conn)
             .unwrap_or_else(|err| die(&pool.logger, "failed to get migration lock", &err));
         // This code can cause a race in database setup: if pool A has had

store/postgres/src/primary.rs

@@ -1519,6 +1519,17 @@ impl<'a> Connection<'a> {
     }
 }
 
+/// Return `true` if we deem this installation to be empty, defined as
+/// having no deployments and no subgraph names in the database
+pub fn is_empty(conn: &PgConnection) -> Result<bool, StoreError> {
+    use deployment_schemas as ds;
+    use subgraph as s;
+
+    let empty = ds::table.count().get_result::<i64>(conn)? == 0
+        && s::table.count().get_result::<i64>(conn)? == 0;
+    Ok(empty)
+}
+
 /// A struct that reads from pools in order, trying each pool in turn until
 /// a query returns either success or anything but a
 /// `Err(StoreError::DatabaseUnavailable)`. This only works for tables that