Commit a9f1a4e8 authored by Alvin Noe Ladines's avatar Alvin Noe Ladines
Browse files

Added xz compression

parent a1c76ed1
Pipeline #62088 failed with stages
in 31 minutes and 58 seconds
...@@ -71,7 +71,7 @@ based on NOMAD-coe's *python-common* module. ...@@ -71,7 +71,7 @@ based on NOMAD-coe's *python-common* module.
:members: :members:
""" """
from typing import Callable, IO, Union from typing import Callable, IO, Union, Dict
import magic import magic
import gzip import gzip
import bz2 import bz2
...@@ -118,7 +118,7 @@ def match_parser(mainfile: str, upload_files: Union[str, files.StagingUploadFile ...@@ -118,7 +118,7 @@ def match_parser(mainfile: str, upload_files: Union[str, files.StagingUploadFile
with open(mainfile_path, 'rb') as f: with open(mainfile_path, 'rb') as f:
compression, open_compressed = _compressions.get(f.read(3), (None, open)) compression, open_compressed = _compressions.get(f.read(3), (None, open))
with open_compressed(mainfile_path, 'rb') as cf: with open_compressed(mainfile_path, 'rb') as cf: # type: ignore
buffer = cf.read(config.parser_matching_size) buffer = cf.read(config.parser_matching_size)
mime_type = magic.from_buffer(buffer, mime=True) mime_type = magic.from_buffer(buffer, mime=True)
......
...@@ -139,8 +139,7 @@ class MatchingParser(Parser): ...@@ -139,8 +139,7 @@ class MatchingParser(Parser):
if self._mainfile_contents_re is not None: if self._mainfile_contents_re is not None:
try: # Try to open the file as a string for regex matching. try: # Try to open the file as a string for regex matching.
decoded_buffer = buffer.decode('utf-8') decoded_buffer = buffer.decode('utf-8')
print('DECODED',decoded_buffer) except UnicodeDecodeError:
except UnicodeDecodeError as e:
return False # We're looking for a string match in a file that can't be converted to string. return False # We're looking for a string match in a file that can't be converted to string.
if self._mainfile_contents_re.search(decoded_buffer) is None: if self._mainfile_contents_re.search(decoded_buffer) is None:
return False return False
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment