I am setting up a local database for datajoint following instructions here: https://github.com/datajoint/mysql-docker.
The mySQL database works but now I wanted to use MinIO for external storage. I fire up MySQL and MinIO as services using docker-compose and set up dj.config for external storage. I am using MinIO service despite running to make it s3 compatible.
Here is my dj.config:
dj.config['stores'] = {
'minio': { # store in s3
'protocol': 's3',
'endpoint': 'minio:9001',
'bucket': 'test',
'location': 'test2/',
'access_key': os.environ.get('MINIO_ROOT_USER', 'FAKEKEY'),
'secret_key': os.environ.get('MINIO_ROOT_PASSWORD', 'FAKEKEY')
}
}
But I get a weird parsing while populating the table. Here is the error stack:
---------------------------------------------------------------------------
ParseError Traceback (most recent call last)
/usr/local/lib/python3.8/dist-packages/minio/parsers.py in fromstring(cls, root_name, data)
68 try:
---> 69 return cls(root_name, ElementTree.fromstring(data.strip()))
70 except (ParseError, AttributeError, ValueError, TypeError) as error:
/usr/lib/python3.8/xml/etree/ElementTree.py in XML(text, parser)
1319 parser = XMLParser(target=TreeBuilder())
-> 1320 parser.feed(text)
1321 return parser.close()
ParseError: syntax error: line 1, column 0
During handling of the above exception, another exception occurred:
InvalidXMLError Traceback (most recent call last)
<ipython-input-24-884d9761b97d> in <module>
----> 1 TrainedModel.populate(reserve_jobs=True)
/usr/local/lib/python3.8/dist-packages/datajoint/autopopulate.py in populate(self, suppress_errors, return_exception_objects, reserve_jobs, order, limit, max_calls, display_progress, *restrictions)
157 self.__class__._allow_insert = True
158 try:
--> 159 make(dict(key))
160 except (KeyboardInterrupt, SystemExit, Exception) as error:
161 try:
/usr/local/lib/python3.8/dist-packages/nnfabrik/templates/trained_model.py in make(self, key)
273 key["model_state"] = filepath
274
--> 275 self.ModelStorage.insert1(key, ignore_extra_fields=True)
276
277
/usr/local/lib/python3.8/dist-packages/datajoint/table.py in insert1(self, row, **kwargs)
173 For kwargs, see insert()
174 """
--> 175 self.insert((row,), **kwargs)
176
177 def insert(self, rows, replace=False, skip_duplicates=False, ignore_extra_fields=False, allow_direct_insert=None):
/usr/local/lib/python3.8/dist-packages/datajoint/table.py in insert(self, rows, replace, skip_duplicates, ignore_extra_fields, allow_direct_insert)
334 return row_to_insert
335
--> 336 rows = list(make_row_to_insert(row) for row in rows)
337 if rows:
338 try:
/usr/local/lib/python3.8/dist-packages/datajoint/table.py in <genexpr>(.0)
334 return row_to_insert
335
--> 336 rows = list(make_row_to_insert(row) for row in rows)
337 if rows:
338 try:
/usr/local/lib/python3.8/dist-packages/datajoint/table.py in make_row_to_insert(row)
304 elif isinstance(row, collections.abc.Mapping): # dict-based
305 check_fields(row)
--> 306 attributes = [make_placeholder(name, row[name]) for name in heading if name in row]
307 else: # positional
308 try:
/usr/local/lib/python3.8/dist-packages/datajoint/table.py in <listcomp>(.0)
304 elif isinstance(row, collections.abc.Mapping): # dict-based
305 check_fields(row)
--> 306 attributes = [make_placeholder(name, row[name]) for name in heading if name in row]
307 else: # positional
308 try:
/usr/local/lib/python3.8/dist-packages/datajoint/table.py in make_placeholder(name, value)
275 if attr.is_external:
276 # value is hash of contents
--> 277 value = self.external[attr.store].upload_attachment(attachment_path).bytes
278 else:
279 # value is filename + contents
/usr/local/lib/python3.8/dist-packages/datajoint/external.py in upload_attachment(self, local_path)
199 uuid = uuid_from_file(local_path, init_string=attachment_name + '\0')
200 external_path = self._make_uuid_path(uuid, '.' + attachment_name)
--> 201 self._upload_file(local_path, external_path)
202 # insert tracking info
203 self.connection.query("""
/usr/local/lib/python3.8/dist-packages/datajoint/external.py in _upload_file(self, local_path, external_path, metadata)
100 def _upload_file(self, local_path, external_path, metadata=None):
101 if self.spec['protocol'] == 's3':
--> 102 self.s3.fput(local_path, external_path, metadata)
103 elif self.spec['protocol'] == 'file':
104 safe_copy(local_path, external_path, overwrite=True)
/usr/local/lib/python3.8/dist-packages/datajoint/external.py in s3(self)
71 def s3(self):
72 if self._s3 is None:
---> 73 self._s3 = s3.Folder(**self.spec)
74 return self._s3
75
/usr/local/lib/python3.8/dist-packages/datajoint/s3.py in __init__(self, endpoint, bucket, access_key, secret_key, secure, **_)
18 secure=secure)
19 self.bucket = bucket
---> 20 if not self.client.bucket_exists(bucket):
21 raise errors.BucketInaccessible('Inaccessible s3 bucket %s' % bucket) from None
22
/usr/local/lib/python3.8/dist-packages/minio/api.py in bucket_exists(self, bucket_name)
402
403 try:
--> 404 self._url_open('HEAD', bucket_name=bucket_name)
405 return True
406 except NoSuchBucket:
/usr/local/lib/python3.8/dist-packages/minio/api.py in _url_open(self, method, bucket_name, object_name, query, body, headers, content_sha256, preload_content)
2183
2184 # Get bucket region.
-> 2185 region = self._get_bucket_region(bucket_name)
2186
2187 # Construct target url.
/usr/local/lib/python3.8/dist-packages/minio/api.py in _get_bucket_region(self, bucket_name)
2061 region = self._region or self._region_map.get(bucket_name)
2062 if not region:
-> 2063 region = self._get_bucket_location(bucket_name)
2064 self._region_map[bucket_name] = region
2065 return region
/usr/local/lib/python3.8/dist-packages/minio/api.py in _get_bucket_location(self, bucket_name)
2105 raise ResponseError(response, method, bucket_name).get_exception()
2106
-> 2107 location = parse_location_constraint(response.data)
2108 # location is empty for 'US standard region'
2109 if not location:
/usr/local/lib/python3.8/dist-packages/minio/parsers.py in parse_location_constraint(data)
425 :return: Returns location of your bucket.
426 """
--> 427 root = S3Element.fromstring('BucketLocationConstraintResult', data)
428 return root.text()
429
/usr/local/lib/python3.8/dist-packages/minio/parsers.py in fromstring(cls, root_name, data)
69 return cls(root_name, ElementTree.fromstring(data.strip()))
70 except (ParseError, AttributeError, ValueError, TypeError) as error:
---> 71 raise InvalidXMLError(
72 '"{}" XML is not parsable. Message: {}'.format(
73 root_name, error
InvalidXMLError: InvalidXMLError: message: "BucketLocationConstraintResult" XML is not parsable. Message: syntax error: line 1, column 0
