From ba69e7c7741ec6735bb6ba5b94855210f71cdf5c Mon Sep 17 00:00:00 2001 From: George Date: Thu, 27 Aug 2026 17:31:24 +0530 Subject: [PATCH] v1 --- .gitignore | 14 ++ comp.txt | 8 ++ instructions.txt | 238 ++++++++++++++++++++++++++++++++++ main.py | 47 +++++++ pgedge.txt | 32 +++++ prometheus/docker-compose.yml | 19 +++ requirements.txt | 3 + 7 files changed, 361 insertions(+) create mode 100644 .gitignore create mode 100644 comp.txt create mode 100644 instructions.txt create mode 100644 main.py create mode 100644 pgedge.txt create mode 100644 prometheus/docker-compose.yml create mode 100644 requirements.txt diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..1e2f68f --- /dev/null +++ b/.gitignore @@ -0,0 +1,14 @@ +# Python virtual environment +venv/ +.venv/ + +# Python cache +__pycache__/ +*.pyc + +# Environment/secrets +.env + +# IDE +.vscode/ +.idea/ diff --git a/comp.txt b/comp.txt new file mode 100644 index 0000000..5e96096 --- /dev/null +++ b/comp.txt @@ -0,0 +1,8 @@ +| Company / Product | True active-active writes | Multi-master | Conflict handling | Monitoring / dashboard | Standard PostgreSQL? | My view | +| -------------------------------------------------------------------------------- | ------------------------- | ------------------------------- | ---------------------------------------- | ---------------------- | ----------------------------- | ----------------------------------------------------- | +| [pgEdge](https://www.pgedge.com/?utm_source=chatgpt.com) | ✅ | ✅ | ✅ | ✅ | ✅ | **Excellent fit** | +| [EDB Postgres Distributed](https://www.enterprisedb.com/?utm_source=chatgpt.com) | ✅ | ✅ | ✅ Advanced | ✅ PEM | ✅ PostgreSQL-based | **Enterprise-grade** | +| [MyDBOps](https://www.mydbops.com/?utm_source=chatgpt.com) | ⚠️ Can design/manage | ⚠️ Depends on technology | Depends | ✅ | ✅ | **Service/consulting provider** | +| Percona | ⚠️ PostgreSQL HA focus | Not its main PostgreSQL product | — | ✅ PMM | ✅ | Better for HA/monitoring than true multi-master | +| YugabyteDB | ✅ | ✅ distributed | ✅ | ✅ | ⚠️ PostgreSQL-compatible | Excellent distributed DB, but not ordinary PostgreSQL | +| CockroachDB | ✅ | ✅ distributed | Built into distributed transaction model | ✅ | ⚠️ PostgreSQL wire compatible | Same caveat: not actual PostgreSQL | diff --git a/instructions.txt b/instructions.txt new file mode 100644 index 0000000..0eec54b --- /dev/null +++ b/instructions.txt @@ -0,0 +1,238 @@ +Let's make a logger. + +CREATE TABLE ddl_history ( + id BIGSERIAL PRIMARY KEY, + event_time TIMESTAMPTZ DEFAULT now(), + command TEXT, + object_type TEXT, + object_name TEXT +); + +Create the function: + +CREATE OR REPLACE FUNCTION capture_ddl() +RETURNS event_trigger +LANGUAGE plpgsql +AS $$ +BEGIN + INSERT INTO ddl_history ( + command, + object_type, + object_name + ) + SELECT + TG_TAG, + object_type, + object_identity + FROM pg_event_trigger_ddl_commands(); +END; +$$; + +DROP TABLE + +CREATE OR REPLACE FUNCTION capture_drop() +RETURNS event_trigger +LANGUAGE plpgsql +AS $$ +BEGIN + INSERT INTO ddl_log ( + event_time, + command, + object_type, + object_name + ) + SELECT + now(), + TG_TAG, + object_type, + object_identity + FROM pg_event_trigger_dropped_objects(); +END; +$$; + + +Create the trigger: + +CREATE EVENT TRIGGER capture_ddl_trigger +ON ddl_command_end +EXECUTE FUNCTION capture_ddl(); + +Now test: + +CREATE TABLE products ( + id INT PRIMARY KEY, + name TEXT +); + +Check: + +SELECT * FROM ddl_history; + + + +docker-compose.yml for LOGICAL REPLICATION MODE + +services: + postgres: + image: postgres:15 + container_name: cluster1Node1 + + restart: unless-stopped + + environment: + POSTGRES_USER: postgres + POSTGRES_PASSWORD: postgres + POSTGRES_DB: postgres + + ports: + - "5444:5432" + + volumes: + - ./postgres_data:/var/lib/postgresql/data + + command: + - postgres + - -c + - wal_level=logical + - -c + - max_replication_slots=10 + - -c + - max_wal_senders=10 + + +sudo docker compose down +sudo docker compose up -d + + +SHOW wal_level; + +psql -h 172.16.10.123 -p 5444 -U postgres + +SHOW max_replication_slots; +SHOW max_wal_senders; + + +CREATE TABLE users ( + id INT PRIMARY KEY, + name TEXT +); + + +SELECT * +FROM pg_create_logical_replication_slot( + 'myslot', + 'test_decoding' +); + +INSERT INTO users VALUES (1, 'Alice'); + +SELECT * +FROM pg_logical_slot_peek_changes( + 'myslot', + NULL, + NULL +); + + +|===================================================| +| psql -h 127.0.0.1 -p 5444 -U postgres -d postgres | +|===================================================| +command: + - postgres + - -c + - wal_level=logical + - -c + - shared_preload_libraries=spock + - -c + - max_replication_slots=10 + - -c + - max_wal_senders=10 + - -c + - track_commit_timestamp=on + +psql -h 127.0.0.1 -p 5444 -U postgres -d postgres + +sudo nano postgres_data/pg_hba.conf + +host all all 172.19.0.0/16 scram-sha-256 +host all all 172.16.10.0/24 scram-sha-256 + +sudo docker exec -it cluster1Node1 psql -U postgres -d postgres +ALTER USER postgres WITH PASSWORD 'postgres'; + +sudo docker exec -it cluster1Node3 bash +psql -U postgres -c "SHOW hba_file;" +psql -U postgres -c "SELECT pg_reload_conf();" + + + + + +|==========| +|LINK Nodes| +|==========| +SELECT spock.sub_create( + subscription_name := 'node2_from_node1', + provider_dsn := 'host=172.16.10.123 port=5444 dbname=postgres user=postgres password=postgres' +); + +|=============| +|Register Node| +|=============| +psql -h localhost -p 5444 -U postgres -d postgres +CREATE EXTENSION IF NOT EXISTS spock; +SELECT extname, extversion +FROM pg_extension +WHERE extname = 'spock'; +SELECT spock.node_create( + node_name := 'node1', + dsn := 'host=172.16.10.123 port=5444 dbname=postgres user=postgres password=postgres' +); +SELECT * FROM spock.node; + +|================| +| CREATE A table | +|================| +CREATE TABLE test_replication ( + id bigint PRIMARY KEY, + source_node text, + message text +); + +SELECT spock.repset_add_table( + 'default', + 'test_replication' +); + +|===========| +|Turn On DDL| +|===========| + +CREATE TABLE customers ( + id bigint PRIMARY KEY, + name text +); +ALTER SYSTEM SET spock.enable_ddl_replication = on; +ALTER SYSTEM SET spock.include_ddl_repset = on; + +SELECT pg_reload_conf(); + +SHOW spock.enable_ddl_replication; +SHOW spock.include_ddl_repset; + +|======================================================| +| HOW TO INSERT OR PUT AUTO GEN OF ID IN EXISTING TABLE| +|======================================================| +ALTER TABLE test_replication +ALTER COLUMN id ADD GENERATED BY DEFAULT AS IDENTITY; + + +HOW TO UPADE A NODE CONFIG + +UPDATE spock.local_node +SET node_local_interface = ( + SELECT if_id + FROM spock.node_interface + WHERE if_name = 'node2_local' + AND if_nodeid = 13534 +); \ No newline at end of file diff --git a/main.py b/main.py new file mode 100644 index 0000000..5e285af --- /dev/null +++ b/main.py @@ -0,0 +1,47 @@ +import psycopg2 + +# ===== TEST DATA ===== +# id = 600 +source_node = "node1" +message = "from python with love " + +conn = None + +try: + conn = psycopg2.connect( + host="172.16.10.123", + port=5444, + database="postgres", + user="postgres", + password="postgres" + ) + + + cursor = conn.cursor() + + cursor.execute(""" + INSERT INTO test_replication (source_node, message) + VALUES (%s, %s) + RETURNING id + """, (source_node, message)) + + # Get the ID generated by PostgreSQL + generated_id = cursor.fetchone()[0] + + conn.commit() + + print("LOCAL COMMIT: SUCCESS") + print("Generated ID:", generated_id) + print("Source:", source_node) + print("Message:", message) + +except Exception as e: + if conn: + conn.rollback() + + print("LOCAL COMMIT: FAILED") + print("Error:", e) + +finally: + if conn: + conn.close() \ No newline at end of file diff --git a/pgedge.txt b/pgedge.txt new file mode 100644 index 0000000..b69a3cf --- /dev/null +++ b/pgedge.txt @@ -0,0 +1,32 @@ +SELECT * FROM spock.node; +output + node_id | node_name | location | country | info +---------+-----------+----------+---------+------ + 30830 | node1 | | | + + +SELECT * FROM spock.node_interface; +output + + if_id | if_name | if_nodeid | if_dsn +------------+---------+-----------+------------------------------------------------------------------------------ + 2198842153 | node3 | 57454 | host=172.16.10.178 port=5444 dbname=postgres user=postgres password=postgres + 2256866226 | node1 | 30830 | host=172.16.10.123 port=5444 dbname=postgres user=postgres password=postgres +(2 rows) + +SELECT * FROM spock.sub_show_status(); +output +| Field | Value | +| --------------------- | ------------------------------------ | +| **Subscription Name** | `sub_node2_from_node1` | +| **Status** | `replicating` | +| **Provider Node** | `node1` | +| **Provider Host** | `172.16.10.123` | +| **Provider Port** | `5444` | +| **Database** | `postgres` | +| **User** | `postgres` | +| **Password** | `postgres` | +| **Slot Name** | `spk_postgres_node1_sub_nodedd485a4` | +| **Replication Sets** | `{default}` | +| **Forward Origins** | empty | + diff --git a/prometheus/docker-compose.yml b/prometheus/docker-compose.yml new file mode 100644 index 0000000..14e3505 --- /dev/null +++ b/prometheus/docker-compose.yml @@ -0,0 +1,19 @@ +services: + prometheus: + image: prom/prometheus:v3.13.2 + container_name: prometheus + restart: unless-stopped + + ports: + - "9090:9090" + + volumes: + - ./prometheus.yml:/etc/prometheus/prometheus.yml:ro + - prometheus_data:/prometheus + + command: + - "--config.file=/etc/prometheus/prometheus.yml" + - "--storage.tsdb.path=/prometheus" + +volumes: + prometheus_data: \ No newline at end of file diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..85676fd --- /dev/null +++ b/requirements.txt @@ -0,0 +1,3 @@ +psycopg==3.3.4 +psycopg-binary==3.3.4 +typing_extensions==4.16.0