Skip to content

Commit

Permalink
Merge pull request #9 from surenpoghosian/package-json-script-modific…
Browse files Browse the repository at this point in the history
…ation

modify npm start logic
  • Loading branch information
surenpoghosian authored Aug 7, 2024
2 parents 87006e7 + 41f1203 commit a73fb38
Show file tree
Hide file tree
Showing 4 changed files with 49 additions and 45 deletions.
2 changes: 1 addition & 1 deletion package.json
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
"main": "index.js",
"type": "module",
"scripts": {
"start": "npx tsx src/Scheduler/index.ts",
"start": "npx tsx src/index.ts",
"start:scheduler": "npx tsx src/Scheduler/index.ts",
"test": "jest --detectOpenHandles",
"build": "tsc",
Expand Down
2 changes: 1 addition & 1 deletion src/Scheduler/index.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
import cron from 'node-cron';
import Main from '..';
import Main from '../main';

class ScraperScheduler {
private cronExpression: string;
Expand Down
46 changes: 3 additions & 43 deletions src/index.ts
Original file line number Diff line number Diff line change
@@ -1,44 +1,4 @@
import { v4 as uuid } from 'uuid';
import ScrapersFactory from './ScrapersFactory';
import ScrapableFactory from './ScrapableFactory';
import PathBuilderFactory from './PathBuilderFactory';
import { ListAmCategory, ListAmGeolocation, PathBuilderVariant, ScrapeValidatorVariant } from "./configs/types";
import { ScrapeableVariant, ScraperType, ScrapeType } from "./configs/types";
import { ListAmBaseURL } from './configs/constants';
import { sleep } from "./utils/sleep";
import ScrapeValidatorFactory from './ScrapeValidatorFactory';
import getRandomInterval from './utils/getRandomInterval';
import Main from "./main";

class Main {
run = async () => {
const scraperFactory = ScrapersFactory;
const scrapableFactory = ScrapableFactory;
const pathBuilderFactory = PathBuilderFactory;
const scrapeValidatorFactory = ScrapeValidatorFactory;

const scraper = scraperFactory.createScraper(ScraperType.PUPPETTER);
const validator = scrapeValidatorFactory.createScrapeValidator(ScrapeValidatorVariant.LISTAM);
const scrapable = scrapableFactory.createScrapable(ScrapeableVariant.LISTAM, scraper, validator);
const pathBuilder = pathBuilderFactory.createPathBuilder(PathBuilderVariant.LISTAM);

pathBuilder.init('', ListAmCategory.ROOM_FOR_A_RENT);

const scrapeId = uuid();

for (let i = 1; true; i++) {
pathBuilder.reset();
pathBuilder.addPageNumber(i);
pathBuilder.addGeolocation(ListAmGeolocation.YEREVAN);

const finalPath = pathBuilder.build();
console.log(`${ListAmBaseURL}${finalPath}`);

await scrapable.scrape(scrapeId, finalPath, ScrapeType.LIST);

const sleepInterval = getRandomInterval(4000, 10000);
await sleep(sleepInterval);
}
}
};

export default Main;
const main = new Main();
main.run();
44 changes: 44 additions & 0 deletions src/main.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,44 @@
import { v4 as uuid } from 'uuid';
import ScrapersFactory from './ScrapersFactory';
import ScrapableFactory from './ScrapableFactory';
import PathBuilderFactory from './PathBuilderFactory';
import { ListAmCategory, ListAmGeolocation, PathBuilderVariant, ScrapeValidatorVariant } from "./configs/types";
import { ScrapeableVariant, ScraperType, ScrapeType } from "./configs/types";
import { ListAmBaseURL } from './configs/constants';
import { sleep } from "./utils/sleep";
import ScrapeValidatorFactory from './ScrapeValidatorFactory';
import getRandomInterval from './utils/getRandomInterval';

class Main {
run = async () => {
const scraperFactory = ScrapersFactory;
const scrapableFactory = ScrapableFactory;
const pathBuilderFactory = PathBuilderFactory;
const scrapeValidatorFactory = ScrapeValidatorFactory;

const scraper = scraperFactory.createScraper(ScraperType.PUPPETTER);
const validator = scrapeValidatorFactory.createScrapeValidator(ScrapeValidatorVariant.LISTAM);
const scrapable = scrapableFactory.createScrapable(ScrapeableVariant.LISTAM, scraper, validator);
const pathBuilder = pathBuilderFactory.createPathBuilder(PathBuilderVariant.LISTAM);

pathBuilder.init('', ListAmCategory.ROOM_FOR_A_RENT);

const scrapeId = uuid();

for (let i = 1; true; i++) {
pathBuilder.reset();
pathBuilder.addPageNumber(i);
pathBuilder.addGeolocation(ListAmGeolocation.YEREVAN);

const finalPath = pathBuilder.build();
console.log(`${ListAmBaseURL}${finalPath}`);

await scrapable.scrape(scrapeId, finalPath, ScrapeType.LIST);

const sleepInterval = getRandomInterval(4000, 10000);
await sleep(sleepInterval);
}
}
};

export default Main;

0 comments on commit a73fb38

Please sign in to comment.