From 7d4948a43b34eccbef596706e85afbbb1ea182e6 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Wed, 20 Sep 2023 22:57:27 +0800 Subject: [PATCH] reduce scroll to 1m --- packages/db/elastic_migrations/migrate_from_elastic.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/db/elastic_migrations/migrate_from_elastic.py b/packages/db/elastic_migrations/migrate_from_elastic.py index 4fbf9e5ff..0f596d6a2 100755 --- a/packages/db/elastic_migrations/migrate_from_elastic.py +++ b/packages/db/elastic_migrations/migrate_from_elastic.py @@ -17,7 +17,7 @@ PG_DB = os.getenv('PG_DB', 'omnivore') ES_URL = os.getenv('ES_URL', 'http://localhost:9200') ES_USERNAME = os.getenv('ES_USERNAME', 'elastic') ES_PASSWORD = os.getenv('ES_PASSWORD', 'password') -ES_SCAN_SIZE = os.getenv('ES_SCAN_SIZE', 10000) +ES_SCAN_SIZE = os.getenv('ES_SCAN_SIZE', 1000) ES_INDEX = os.getenv('ES_INDEX', 'pages_alias') CUT_OFF_DATE = os.getenv('CUT_OFF_DATE', '2000-01-01') @@ -309,7 +309,7 @@ async def main(): # Scan API for larger library docs = async_scan(es_client, index=ES_INDEX, query=query, preserve_order=True, size=ES_SCAN_SIZE, - request_timeout=600) + request_timeout=60, scroll='1m') # convert _id to uuid async for doc in docs: