Skip to content

Commit

Permalink
queue@7
Browse files Browse the repository at this point in the history
  • Loading branch information
derhuerst committed Sep 5, 2023
1 parent 054a297 commit 27d0c8c
Show file tree
Hide file tree
Showing 2 changed files with 24 additions and 21 deletions.
43 changes: 23 additions & 20 deletions build/index.js
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
import createQueue from 'queue'
import Queue from 'queue'
import pick from 'lodash.pick'
import {join as pathJoin} from 'node:path'
import {writeFile} from 'node:fs'
Expand All @@ -12,40 +12,41 @@ const getNrOfPages = async () => {

const fetchAll = (nrOfPages) => {
return new Promise((yay, nay) => {
const queue = createQueue({concurrency: 2, autostart: true})
const queue = new Queue({concurrency: 2, autostart: true})
let results = []

const fetch = (i) => {
const job = (cb) => {
req({
const job = async () => {
const data = await req({
q: '',
// page=0 fails with 422 for some reason 🙄
page: i === 0 ? undefined : i,
})
.then((data) => {
results = results.concat(data.results)
cb()
})
.catch(cb)
results = [
...results,
...data.results,
]
}

job.title = i + ''
return job
}

queue.once('error', (err) => {
queue.addEventListener('error', (ev) => {
const {error: err} = ev.detail
queue.stop()
nay(err)
})
queue.once('end', (err) => {
}, {once: true})
queue.addEventListener('end', (ev) => {
const {error: err} = ev.detail
if (!err) yay(results)
})
queue.on('success', (_, job) => {
}, {once: true})
queue.addEventListener('success', (ev) => {
const {job} = ev.detail
console.log(job.title + '/' + nrOfPages)
})

for (let i = 0; i <= nrOfPages; i++) {
// for (let i = 0; i <= 10; i++) { // todo
queue.push(fetch(i))
}
})
Expand All @@ -68,7 +69,7 @@ const dir = fileURLToPath(new URL('../s', import.meta.url).href)

const storeIndividuals = (index) => {
return new Promise((yay, nay) => {
const queue = createQueue({concurrency: 8, autostart: true})
const queue = new Queue({concurrency: 8, autostart: true})

const store = (result) => {
const job = (cb) => {
Expand All @@ -80,13 +81,15 @@ const storeIndividuals = (index) => {
return job
}

queue.once('error', (err) => {
queue.addEventListener('error', (ev) => {
const {err} = ev.detail
queue.stop()
nay(err)
})
queue.once('end', (err) => {
}, {once: true})
queue.addEventListener('end', (ev) => {
const {err} = ev.detail
if (!err) yay()
})
}, {once: true})

for (let result of index) {
queue.push(store(result))
Expand Down
2 changes: 1 addition & 1 deletion package.json
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@
"lodash.pick": "^4.4.0",
"p-retry": "^4.6.2",
"qs": "^6.5.0",
"queue": "^6.0.0",
"queue": "^7.0.0",
"tap-min": "^2.0.0",
"tape": "^5.0.0"
},
Expand Down

0 comments on commit 27d0c8c

Please sign in to comment.