close writestream on finish

This commit is contained in:
Hongbo Wu
2024-07-19 21:44:54 +08:00
parent 01bce43781
commit f749090a7d

View File

@ -81,22 +81,31 @@ export const exporter = Sentry.GCPFunction.wrapHttpFunction(
}) })
try { try {
// write the list of urls to a csv file and upload it to gcs // write the exported data to a csv file and upload it to gcs
// path style: exports/<uid>/<date>/<uuid>.csv // path style: exports/<uid>/<date>/<uuid>.csv
const dateStr = new Date().toISOString() const dateStr = new Date().toISOString()
const fileUuid = uuidv4() const fileUuid = uuidv4()
const fullPath = `exports/${claims.uid}/${dateStr}/${fileUuid}.csv` const fullPath = `exports/${claims.uid}/${dateStr}/${fileUuid}.csv`
// open a write_stream to the file
const file = createGCSFile(GCS_BUCKET, fullPath) const file = createGCSFile(GCS_BUCKET, fullPath)
const writeStream = file.createWriteStream({
contentType: 'text/csv',
})
// stringify the data and pipe it to the write_stream // stringify the data and pipe it to the write_stream
const stringifier = stringify({ const stringifier = stringify({
header: true, header: true,
columns: ['url', 'state', 'labels'], columns: ['id', 'title', 'description', 'state', 'labels'],
}) })
stringifier.pipe(writeStream)
stringifier
.pipe(
file.createWriteStream({
contentType: 'text/csv',
})
)
.on('error', (err) => {
console.error('error writing to file', err)
})
.on('finish', () => {
console.log('done writing to file')
})
// fetch data from the database // fetch data from the database
const omnivore = new Omnivore({ const omnivore = new Omnivore({
@ -122,7 +131,7 @@ export const exporter = Sentry.GCPFunction.wrapHttpFunction(
} }
} while (cursor) } while (cursor)
writeStream.end() stringifier.end()
// generate a temporary signed url for the csv file // generate a temporary signed url for the csv file
const signedUrl = await createSignedUrl(file) const signedUrl = await createSignedUrl(file)