I'm getting the following error trying to import a JSON file. I tried it with the latest elastic search 8.5.3 and also the last tested/supported version that the loader was tested with (8.1.2):
Stevens-MBP:defect-tracking smc2$ elasticsearch_loader --ca-certs certs --http-auth elastic:mypassword --index steve-dec12c --es-host https://ivtscenarios1.fyre.ibm.com:9200 json steve2.json --lines
/opt/homebrew/lib/python3.10/site-packages/elasticsearch/connection/http_urllib3.py:209: UserWarning: Connecting to https://ivtscenarios1.fyre.ibm.com:9200 using SSL with verify_certs=False is insecure.
warnings.warn(
{'ca_certs': 'certs', 'http_auth': 'elastic:cpd2cool', 'index': 'steve-dec12c', 'es_host': ('https://ivtscenarios1.fyre.ibm.com:9200',), 'bulk_size': 500, 'verify_certs': False, 'use_ssl': False, 'delete': False, 'update': False, 'progress': False, 'type': '_doc', 'id_field': None, 'as_child': False, 'with_retry': False, 'index_settings_file': None, 'timeout': 10.0, 'encoding': 'utf-8', 'keys': [], 'es_conn': <Elasticsearch([{'host': 'ivtscenarios1.fyre.ibm.com', 'port': 9200, 'use_ssl': True}])>}
/opt/homebrew/lib/python3.10/site-packages/urllib3/connectionpool.py:1045: InsecureRequestWarning: Unverified HTTPS request is being made to host 'ivtscenarios1.fyre.ibm.com'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings
warnings.warn(
/opt/homebrew/lib/python3.10/site-packages/urllib3/connectionpool.py:1045: InsecureRequestWarning: Unverified HTTPS request is being made to host 'ivtscenarios1.fyre.ibm.com'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings
warnings.warn(
/opt/homebrew/lib/python3.10/site-packages/urllib3/connectionpool.py:1045: InsecureRequestWarning: Unverified HTTPS request is being made to host 'ivtscenarios1.fyre.ibm.com'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings
warnings.warn(
/opt/homebrew/lib/python3.10/site-packages/urllib3/connectionpool.py:1045: InsecureRequestWarning: Unverified HTTPS request is being made to host 'ivtscenarios1.fyre.ibm.com'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings
warnings.warn(
2022-12-13 21:54:08.035595 ERROR attempt [1/1] got exception, it is a permanent data loss, no retry any more
2022-12-13 21:54:08.035788 WARN Chunk 0 got exception (RequestError(400, 'illegal_argument_exception', 'Action/metadata line [1] contains an unknown parameter [_type]')) while processing
Traceback (most recent call last):
File "/opt/homebrew/bin/elasticsearch_loader", line 8, in <module>
sys.exit(cli())
File "/opt/homebrew/lib/python3.10/site-packages/click/core.py", line 764, in __call__
return self.main(*args, **kwargs)
File "/opt/homebrew/lib/python3.10/site-packages/click/core.py", line 717, in main
rv = self.invoke(ctx)
File "/opt/homebrew/lib/python3.10/site-packages/click/core.py", line 1137, in invoke
return _process_result(sub_ctx.command.invoke(sub_ctx))
File "/opt/homebrew/lib/python3.10/site-packages/click/core.py", line 956, in invoke
return ctx.invoke(self.callback, **ctx.params)
File "/opt/homebrew/lib/python3.10/site-packages/click/core.py", line 555, in invoke
return callback(*args, **kwargs)
File "/opt/homebrew/lib/python3.10/site-packages/click/decorators.py", line 17, in new_func
return f(get_current_context(), *args, **kwargs)
File "/opt/homebrew/lib/python3.10/site-packages/elasticsearch_loader/__init__.py", line 150, in _json
load(lines, ctx.obj)
File "/opt/homebrew/lib/python3.10/site-packages/elasticsearch_loader/__init__.py", line 53, in load
single_bulk_to_es(bulk, config, config['with_retry'])
File "/opt/homebrew/lib/python3.10/site-packages/elasticsearch_loader/__init__.py", line 37, in single_bulk_to_es
raise e
File "/opt/homebrew/lib/python3.10/site-packages/elasticsearch_loader/__init__.py", line 28, in single_bulk_to_es
helpers.bulk(config['es_conn'], bulk, chunk_size=config['bulk_size'])
File "/opt/homebrew/lib/python3.10/site-packages/elasticsearch/helpers/actions.py", line 410, in bulk
for ok, item in streaming_bulk(
File "/opt/homebrew/lib/python3.10/site-packages/elasticsearch/helpers/actions.py", line 329, in streaming_bulk
for data, (ok, info) in zip(
File "/opt/homebrew/lib/python3.10/site-packages/elasticsearch/helpers/actions.py", line 256, in _process_bulk_chunk
for item in gen:
File "/opt/homebrew/lib/python3.10/site-packages/elasticsearch/helpers/actions.py", line 195, in _process_bulk_chunk_error
raise error
File "/opt/homebrew/lib/python3.10/site-packages/elasticsearch/helpers/actions.py", line 240, in _process_bulk_chunk
resp = client.bulk("\n".join(bulk_actions) + "\n", *args, **kwargs)
File "/opt/homebrew/lib/python3.10/site-packages/elasticsearch/client/utils.py", line 168, in _wrapped
return func(*args, params=params, headers=headers, **kwargs)
File "/opt/homebrew/lib/python3.10/site-packages/elasticsearch/client/__init__.py", line 463, in bulk
return self.transport.perform_request(
File "/opt/homebrew/lib/python3.10/site-packages/elasticsearch/transport.py", line 458, in perform_request
raise e
File "/opt/homebrew/lib/python3.10/site-packages/elasticsearch/transport.py", line 419, in perform_request
status, headers_response, data = connection.perform_request(
File "/opt/homebrew/lib/python3.10/site-packages/elasticsearch/connection/http_urllib3.py", line 277, in perform_request
self._raise_error(response.status, raw_data)
File "/opt/homebrew/lib/python3.10/site-packages/elasticsearch/connection/base.py", line 330, in _raise_error
raise HTTP_EXCEPTIONS.get(status_code, TransportError)(
elasticsearch.exceptions.RequestError: RequestError(400, 'illegal_argument_exception', 'Action/metadata line [1] contains an unknown parameter [_type]')
Stevens-MBP:defect-tracking smc2$
Stevens-MBP:defect-tracking smc2$ elasticsearch_loader --ca-certs certs --verify-certs --http-auth elastic:mypassword --index steve-dec12c --es-host https://ivtscenarios1.fyre.ibm.com:9200 json steve2.json --lines
{'ca_certs': 'certs', 'verify_certs': True, 'http_auth': 'elastic:cpd2cool', 'index': 'steve-dec12c', 'es_host': ('https://ivtscenarios1.fyre.ibm.com:9200',), 'bulk_size': 500, 'use_ssl': False, 'delete': False, 'update': False, 'progress': False, 'type': '_doc', 'id_field': None, 'as_child': False, 'with_retry': False, 'index_settings_file': None, 'timeout': 10.0, 'encoding': 'utf-8', 'keys': [], 'es_conn': <Elasticsearch([{'host': 'ivtscenarios1.fyre.ibm.com', 'port': 9200, 'use_ssl': True}])>}
2022-12-13 21:56:22.376659 ERROR attempt [1/1] got exception, it is a permanent data loss, no retry any more
2022-12-13 21:56:22.376786 WARN Chunk 0 got exception (ConnectionError([Errno 21] Is a directory) caused by: SSLError([Errno 21] Is a directory)) while processing
Traceback (most recent call last):
File "/opt/homebrew/lib/python3.10/site-packages/urllib3/util/ssl_.py", line 402, in ssl_wrap_socket
context.load_verify_locations(ca_certs, ca_cert_dir, ca_cert_data)
IsADirectoryError: [Errno 21] Is a directory
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/opt/homebrew/lib/python3.10/site-packages/elasticsearch/connection/http_urllib3.py", line 251, in perform_request
response = self.pool.urlopen(
File "/opt/homebrew/lib/python3.10/site-packages/urllib3/connectionpool.py", line 787, in urlopen
retries = retries.increment(
File "/opt/homebrew/lib/python3.10/site-packages/urllib3/util/retry.py", line 525, in increment
raise six.reraise(type(error), error, _stacktrace)
File "/opt/homebrew/lib/python3.10/site-packages/urllib3/packages/six.py", line 770, in reraise
raise value
File "/opt/homebrew/lib/python3.10/site-packages/urllib3/connectionpool.py", line 703, in urlopen
httplib_response = self._make_request(
File "/opt/homebrew/lib/python3.10/site-packages/urllib3/connectionpool.py", line 386, in _make_request
self._validate_conn(conn)
File "/opt/homebrew/lib/python3.10/site-packages/urllib3/connectionpool.py", line 1042, in _validate_conn
conn.connect()
File "/opt/homebrew/lib/python3.10/site-packages/urllib3/connection.py", line 414, in connect
self.sock = ssl_wrap_socket(
File "/opt/homebrew/lib/python3.10/site-packages/urllib3/util/ssl_.py", line 404, in ssl_wrap_socket
raise SSLError(e)
urllib3.exceptions.SSLError: [Errno 21] Is a directory
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/opt/homebrew/bin/elasticsearch_loader", line 8, in <module>
sys.exit(cli())
File "/opt/homebrew/lib/python3.10/site-packages/click/core.py", line 764, in __call__
return self.main(*args, **kwargs)
File "/opt/homebrew/lib/python3.10/site-packages/click/core.py", line 717, in main
rv = self.invoke(ctx)
File "/opt/homebrew/lib/python3.10/site-packages/click/core.py", line 1137, in invoke
return _process_result(sub_ctx.command.invoke(sub_ctx))
File "/opt/homebrew/lib/python3.10/site-packages/click/core.py", line 956, in invoke
return ctx.invoke(self.callback, **ctx.params)
File "/opt/homebrew/lib/python3.10/site-packages/click/core.py", line 555, in invoke
return callback(*args, **kwargs)
File "/opt/homebrew/lib/python3.10/site-packages/click/decorators.py", line 17, in new_func
return f(get_current_context(), *args, **kwargs)
File "/opt/homebrew/lib/python3.10/site-packages/elasticsearch_loader/__init__.py", line 150, in _json
load(lines, ctx.obj)
File "/opt/homebrew/lib/python3.10/site-packages/elasticsearch_loader/__init__.py", line 53, in load
single_bulk_to_es(bulk, config, config['with_retry'])
File "/opt/homebrew/lib/python3.10/site-packages/elasticsearch_loader/__init__.py", line 37, in single_bulk_to_es
raise e
File "/opt/homebrew/lib/python3.10/site-packages/elasticsearch_loader/__init__.py", line 28, in single_bulk_to_es
helpers.bulk(config['es_conn'], bulk, chunk_size=config['bulk_size'])
File "/opt/homebrew/lib/python3.10/site-packages/elasticsearch/helpers/actions.py", line 410, in bulk
for ok, item in streaming_bulk(
File "/opt/homebrew/lib/python3.10/site-packages/elasticsearch/helpers/actions.py", line 329, in streaming_bulk
for data, (ok, info) in zip(
File "/opt/homebrew/lib/python3.10/site-packages/elasticsearch/helpers/actions.py", line 256, in _process_bulk_chunk
for item in gen:
File "/opt/homebrew/lib/python3.10/site-packages/elasticsearch/helpers/actions.py", line 195, in _process_bulk_chunk_error
raise error
File "/opt/homebrew/lib/python3.10/site-packages/elasticsearch/helpers/actions.py", line 240, in _process_bulk_chunk
resp = client.bulk("\n".join(bulk_actions) + "\n", *args, **kwargs)
File "/opt/homebrew/lib/python3.10/site-packages/elasticsearch/client/utils.py", line 168, in _wrapped
return func(*args, params=params, headers=headers, **kwargs)
File "/opt/homebrew/lib/python3.10/site-packages/elasticsearch/client/__init__.py", line 463, in bulk
return self.transport.perform_request(
File "/opt/homebrew/lib/python3.10/site-packages/elasticsearch/transport.py", line 409, in perform_request
self._do_verify_elasticsearch(headers=headers, timeout=timeout)
File "/opt/homebrew/lib/python3.10/site-packages/elasticsearch/transport.py", line 598, in _do_verify_elasticsearch
raise error
File "/opt/homebrew/lib/python3.10/site-packages/elasticsearch/transport.py", line 561, in _do_verify_elasticsearch
_, info_headers, info_response = conn.perform_request(
File "/opt/homebrew/lib/python3.10/site-packages/elasticsearch/connection/http_urllib3.py", line 263, in perform_request
raise SSLError("N/A", str(e), e)
elasticsearch.exceptions.SSLError: ConnectionError([Errno 21] Is a directory) caused by: SSLError([Errno 21] Is a directory)
Stevens-MBP:defect-tracking smc2$
I'm getting the following error trying to import a JSON file. I tried it with the latest elastic search 8.5.3 and also the last tested/supported version that the loader was tested with (8.1.2):
Also, if I add the --verify-certs option, then I get this error: