Skip to content

Commit

Permalink
Supporting postgressql as well
Browse files Browse the repository at this point in the history
  • Loading branch information
rahulbansalc6414 committed Jan 29, 2025
1 parent 78f5df1 commit 3e67aa8
Show file tree
Hide file tree
Showing 4 changed files with 87 additions and 23 deletions.
2 changes: 2 additions & 0 deletions .env.example
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
IS_DB_MYSQL=true
IS_DB_POSTGRES=false
DB_HOST=1.2.3.4
DB_USER=a
DB_PASSWORD=a
Expand Down
4 changes: 2 additions & 2 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ This Node.js project analyzes the null distribution of columns in a specified da

## Features

- Calculates the distribution of null values for each column.
- You just need to provide the table name, it will automatically calculate the distribution of null values for every column.
- Sorts columns by the number of null values in descending order.
- Intelligently limits the no_of_rows to be scanned, configurable via `DEFAULT_LIMIT`
- Outputs results in JSON or HTML format.
Expand All @@ -13,7 +13,7 @@ This Node.js project analyzes the null distribution of columns in a specified da
## Requirements

- Node.js (v14 or higher)
- A relational database (only mysql for now)
- A relational database (it works with mysql/postgresql)

## Installation

Expand Down
101 changes: 81 additions & 20 deletions app.js
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
// Import necessary modules
const express = require('express');
const mysql = require('mysql2/promise');
const { Client } = require('pg');
const dotenv = require('dotenv');
const path = require('path');

Expand All @@ -22,6 +23,77 @@ const dbConfig = {
};

const DEFAULT_LIMIT = parseInt(process.env.DEFAULT_LIMIT, 10) || 100;
const isDbMysql = process.env.IS_DB_MYSQL === 'true';
const isDbPostgres = process.env.IS_DB_POSTGRES === 'true';


/**
* Fetch data from MySQL database.
* @param {string} table - Table name.
* @param {number} limit - Number of rows to fetch.
* @returns {Array} - Rows from the table.
*/
async function getDataFromMysql(table, limit)
{
const connection = await mysql.createConnection(dbConfig);

// Fetch the primary key column
const [primaryKeyResult] = await connection.query(
`SHOW KEYS FROM \`${table}\` WHERE Key_name = 'PRIMARY'`
);
if (!primaryKeyResult.length)
{
throw new Error(`No primary key found for table '${table}'.`);
}
const primaryKey = primaryKeyResult[0].Column_name;

// Fetch data from the table
const [rows] = await connection.query(
`SELECT * FROM \`${table}\` ORDER BY \`${primaryKey}\` DESC LIMIT ?`,
[limit]
);
await connection.end();
return rows;
}/**
* Fetch data from PostgreSQL database.
* @param {string} table - Table name.
* @param {number} limit - Number of rows to fetch.
* @returns {Array} - Rows from the table.
*/
async function getDataFromPostgres(table, limit)
{
const client = new Client({
...dbConfig, ssl: {
rejectUnauthorized: false
}
})

await client.connect();

// Fetch the primary key column
const primaryKeyResult = await client.query(
`SELECT a.attname
FROM pg_index i
JOIN pg_attribute a ON a.attrelid = i.indrelid AND a.attnum = ANY(i.indkey)
WHERE i.indrelid = '${table}'::regclass AND i.indisprimary`
);

if (!primaryKeyResult.rows.length)
{
throw new Error(`No primary key found for table '${table}'.`);
}

const primaryKey = primaryKeyResult.rows[0].attname;

// Fetch data from the table
const res = await client.query(
`SELECT * FROM ${table} ORDER BY ${primaryKey} DESC LIMIT $1`,
[limit]
);

await client.end();
return res.rows;
}

/**
* Fetch null distribution for a given table.
Expand All @@ -31,27 +103,19 @@ const DEFAULT_LIMIT = parseInt(process.env.DEFAULT_LIMIT, 10) || 100;
*/
async function getNullDistribution(table, limit)
{
let connection;
try
{
// Create a database connection
connection = await mysql.createConnection(dbConfig);

// Fetch the primary key column
const [primaryKeyResult] = await connection.query(
`SHOW KEYS FROM \`${table}\` WHERE Key_name = 'PRIMARY'`
);
if (!primaryKeyResult.length)
let rows;
if (isDbMysql)
{
rows = await getDataFromMysql(table, limit);
} else if (isDbPostgres)
{
throw new Error(`No primary key found for table '${table}'.`);
rows = await getDataFromPostgres(table, limit);
} else
{
throw new Error('No valid database configuration found.');
}
const primaryKey = primaryKeyResult[0].Column_name;

// Fetch data from the table
const [rows] = await connection.query(
`SELECT * FROM \`${table}\` ORDER BY \`${primaryKey}\` DESC LIMIT ?`,
[limit]
);

if (!rows.length)
{
Expand Down Expand Up @@ -114,9 +178,6 @@ async function getNullDistribution(table, limit)
{
console.error('Error:', error);
throw error;
} finally
{
if (connection) await connection.end();
}
}

Expand Down
3 changes: 2 additions & 1 deletion package.json
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@
"dotenv": "^16.4.7",
"ejs": "^3.1.10",
"express": "^4.21.2",
"mysql2": "^3.12.0"
"mysql2": "^3.12.0",
"pg": "^8.13.1"
}
}

0 comments on commit 3e67aa8

Please sign in to comment.