Too many SQL variables error
Caught an error on one background task:
Traceback (most recent call last):
File "/appl/soft/4.0.0b4/lib64/python3.9/site-packages/SQLAlchemy-2.0.27-py3.9.egg/sqlalchemy/engine/base.py", line 1960, in _exec_single_context
self.dialect.do_execute(
File "/appl/soft/4.0.0b4/lib64/python3.9/site-packages/SQLAlchemy-2.0.27-py3.9.egg/sqlalchemy/engine/default.py", line 924, in do_execute
cursor.execute(statement, parameters)
sqlite3.OperationalError: too many SQL variables
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "/appl/soft/4.0.0b4/lib64/python3.9/site-packages/autosubmit_api-4.0.0b4-py3.9.egg/autosubmit_api/bgtasks/bgtask.py", line 23, in run
cls.procedure()
File "/appl/soft/4.0.0b4/lib64/python3.9/site-packages/autosubmit_api-4.0.0b4-py3.9.egg/autosubmit_api/bgtasks/bgtask.py", line 51, in procedure
return DetailsProcessor(APIBasicConfig).process()
File "/appl/soft/4.0.0b4/lib64/python3.9/site-packages/autosubmit_api-4.0.0b4-py3.9.egg/autosubmit_api/workers/populate_details/populate.py", line 33, in process
return self._insert_many_into_details_table(new_details)
File "/appl/soft/4.0.0b4/lib64/python3.9/site-packages/autosubmit_api-4.0.0b4-py3.9.egg/autosubmit_api/workers/populate_details/populate.py", line 90, in _insert_many_into_details_table
result = conn.execute(tables.details_table.insert().values(values))
File "/appl/soft/4.0.0b4/lib64/python3.9/site-packages/SQLAlchemy-2.0.27-py3.9.egg/sqlalchemy/engine/base.py", line 1408, in execute
return meth(
File "/appl/soft/4.0.0b4/lib64/python3.9/site-packages/SQLAlchemy-2.0.27-py3.9.egg/sqlalchemy/sql/elements.py", line 513, in _execute_on_connection
return connection._execute_clauseelement(
File "/appl/soft/4.0.0b4/lib64/python3.9/site-packages/SQLAlchemy-2.0.27-py3.9.egg/sqlalchemy/engine/base.py", line 1630, in _execute_clauseelement
ret = self._execute_context(
File "/appl/soft/4.0.0b4/lib64/python3.9/site-packages/SQLAlchemy-2.0.27-py3.9.egg/sqlalchemy/engine/base.py", line 1839, in _execute_context
return self._exec_single_context(
File "/appl/soft/4.0.0b4/lib64/python3.9/site-packages/SQLAlchemy-2.0.27-py3.9.egg/sqlalchemy/engine/base.py", line 1979, in _exec_single_context
self._handle_dbapi_exception(
File "/appl/soft/4.0.0b4/lib64/python3.9/site-packages/SQLAlchemy-2.0.27-py3.9.egg/sqlalchemy/engine/base.py", line 2335, in _handle_dbapi_exception
raise sqlalchemy_exception.with_traceback(exc_info[2]) from e
File "/appl/soft/4.0.0b4/lib64/python3.9/site-packages/SQLAlchemy-2.0.27-py3.9.egg/sqlalchemy/engine/base.py", line 1960, in _exec_single_context
self.dialect.do_execute(
File "/appl/soft/4.0.0b4/lib64/python3.9/site-packages/SQLAlchemy-2.0.27-py3.9.egg/sqlalchemy/engine/default.py", line 924, in do_execute
cursor.execute(statement, parameters)
sqlalchemy.exc.OperationalError: (sqlite3.OperationalError) too many SQL variables
[SQL: INSERT INTO details (exp_id, user, created, model, branch, hpc) VALUES (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?)]
[parameters: (11, 'agayayav', '2023-04-13 17:34:05', 'https://earth.bsc.es/gitlab/digital-twins/de_340/workflow.git', 'vanilla_workflow_handson', 'MARENOSTRUM4', 21, 'mandresm', '2023-04-17 17:25:27', 'https://earth.bsc.es/gitlab/digital-twins/de_340/workflow.git', 'vanilla_workflow_handson', 'LUMI', 34, 'maxness', '2023-04-19 12:09:26', 'https://earth.bsc.es/gitlab/digital-twins/de_340/workflow.git', 'vanilla_workflow_handson', 'LUMI', 35, 'lautuppi', '2023-04-19 12:09:42', 'https://earth.bsc.es/gitlab/digital-twins/de_340/workflow.git', 'vanilla_workflow_handson', 'LEVANTE', 36, 'mnurisso', '2023-04-19 12:09:55', 'https://earth.bsc.es/gitlab/digital-twins/de_340/workflow.git', 'vanilla_workflow_handson', 'LEVANTE', 37, 'backman', '2023-04-19 12:10:55', 'https://earth.bsc.es/gitlab/digital-twins/de_340/workflow.git', 'vanilla_workflow_handson', 'LUMI', 40, 'amohamme', '2023-04-19 12:09:32', 'https://earth.bsc.es/gitlab/digital-twins/de_340/workflow.git', 'vanilla_workflow_handson', 'LUMI', 41, 'jlento', '2023-04-19 12:08:58', 'https://earth.bsc.es/gitlab/digital-twins/de_340/workflow.git', 'vanilla_workflow_handson', 'LUMI', 42, 'kkeller' ... 1526 parameters truncated ... '215-fix-dqc-transfer-jobs', 'LUMI', 1268, 'esharifi', '2024-02-23 13:02:03', 'https://earth.bsc.es/gitlab/digital-twins/de_340/workflow', 'Ehsan_test_branch', 'LUMI', 1269, 'agayayav', '2024-02-23 14:04:14', 'https://earth.bsc.es/gitlab/digital-twins/de_340/workflow', '215-fix-dqc-transfer-jobs', 'LUMI', 1271, 'nrocha', '2024-02-23 17:05:30', 'https://earth.bsc.es/gitlab/digital-twins/de_340/workflow', 'feature/ensambles', 'LUMI', 1273, 'kkeller', '2024-02-24 15:34:54', 'https://earth.bsc.es/gitlab/digital-twins/de_340/workflow', 'feature/ensambles', 'LUMI', 1275, 'agayayav', '2024-02-26 13:28:02', 'https://earth.bsc.es/gitlab/digital-twins/de_340/workflow', 'update-aqua', 'LUMI', 1277, 'sbeyer', '2024-02-26 14:58:25', 'https://earth.bsc.es/gitlab/digital-twins/de_340/workflow', 'main', 'LUMI', 1278, 'jlehtira', '2024-02-26 16:21:59', 'https://earth.bsc.es/gitlab/digital-twins/de_340/workflow', 'app-subworkflow', 'LUMI', 1280, 'redler2', '2024-02-27 10:43:56', 'https://earth.bsc.es/gitlab/digital-twins/de_340/workflow', 'main', 'LUMI')]
(Background on this error at: https://sqlalche.me/e/20/e3q8)
ERROR:gunicorn.error:NoneUsually, DBMS handles variable limits that can be declared in a single transaction. In the case of Sqlite, it is by default 999 (https://www.sqlite.org/limits.html), and is not a good practice to modify it to solve this issue.