Benature/omnivore
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

reduce scroll to 1m

Browse Source
This commit is contained in:
Hongbo Wu
2023-09-20 22:57:27 +08:00
parent 8f1664b058
commit 7d4948a43b
1 changed files with 2 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
packages/db/elastic_migrations/migrate_from_elastic.py
View File

@ -17,7 +17,7 @@ PG_DB = os.getenv('PG_DB', 'omnivore')
ES_URL = os.getenv('ES_URL', 'http://localhost:9200')
ES_USERNAME = os.getenv('ES_USERNAME', 'elastic')
ES_PASSWORD = os.getenv('ES_PASSWORD', 'password')
ES_SCAN_SIZE = os.getenv('ES_SCAN_SIZE', 10000)
ES_SCAN_SIZE = os.getenv('ES_SCAN_SIZE', 1000)
ES_INDEX = os.getenv('ES_INDEX', 'pages_alias')
CUT_OFF_DATE = os.getenv('CUT_OFF_DATE', '2000-01-01')
@ -309,7 +309,7 @@ async def main():
# Scan API for larger library
docs = async_scan(es_client, index=ES_INDEX, query=query,
preserve_order=True, size=ES_SCAN_SIZE,
request_timeout=600)
request_timeout=60, scroll='1m')
# convert _id to uuid
async for doc in docs: