close writestream on finish
This commit is contained in:
@ -81,22 +81,31 @@ export const exporter = Sentry.GCPFunction.wrapHttpFunction(
|
|||||||
})
|
})
|
||||||
|
|
||||||
try {
|
try {
|
||||||
// write the list of urls to a csv file and upload it to gcs
|
// write the exported data to a csv file and upload it to gcs
|
||||||
// path style: exports/<uid>/<date>/<uuid>.csv
|
// path style: exports/<uid>/<date>/<uuid>.csv
|
||||||
const dateStr = new Date().toISOString()
|
const dateStr = new Date().toISOString()
|
||||||
const fileUuid = uuidv4()
|
const fileUuid = uuidv4()
|
||||||
const fullPath = `exports/${claims.uid}/${dateStr}/${fileUuid}.csv`
|
const fullPath = `exports/${claims.uid}/${dateStr}/${fileUuid}.csv`
|
||||||
// open a write_stream to the file
|
|
||||||
const file = createGCSFile(GCS_BUCKET, fullPath)
|
const file = createGCSFile(GCS_BUCKET, fullPath)
|
||||||
const writeStream = file.createWriteStream({
|
|
||||||
contentType: 'text/csv',
|
|
||||||
})
|
|
||||||
// stringify the data and pipe it to the write_stream
|
// stringify the data and pipe it to the write_stream
|
||||||
const stringifier = stringify({
|
const stringifier = stringify({
|
||||||
header: true,
|
header: true,
|
||||||
columns: ['url', 'state', 'labels'],
|
columns: ['id', 'title', 'description', 'state', 'labels'],
|
||||||
})
|
})
|
||||||
stringifier.pipe(writeStream)
|
|
||||||
|
stringifier
|
||||||
|
.pipe(
|
||||||
|
file.createWriteStream({
|
||||||
|
contentType: 'text/csv',
|
||||||
|
})
|
||||||
|
)
|
||||||
|
.on('error', (err) => {
|
||||||
|
console.error('error writing to file', err)
|
||||||
|
})
|
||||||
|
.on('finish', () => {
|
||||||
|
console.log('done writing to file')
|
||||||
|
})
|
||||||
|
|
||||||
// fetch data from the database
|
// fetch data from the database
|
||||||
const omnivore = new Omnivore({
|
const omnivore = new Omnivore({
|
||||||
@ -122,7 +131,7 @@ export const exporter = Sentry.GCPFunction.wrapHttpFunction(
|
|||||||
}
|
}
|
||||||
} while (cursor)
|
} while (cursor)
|
||||||
|
|
||||||
writeStream.end()
|
stringifier.end()
|
||||||
|
|
||||||
// generate a temporary signed url for the csv file
|
// generate a temporary signed url for the csv file
|
||||||
const signedUrl = await createSignedUrl(file)
|
const signedUrl = await createSignedUrl(file)
|
||||||
|
|||||||
Reference in New Issue
Block a user