diff --git a/nomad/app/dcat/common.py b/nomad/app/dcat/common.py index 3d782dad8884a344cad2a4dbe6e66af47bc68e4b..90e6fc4876220a103f264a83c77806d0d34e9391 100644 --- a/nomad/app/dcat/common.py +++ b/nomad/app/dcat/common.py @@ -16,13 +16,13 @@ # limitations under the License. # -from fastapi import Response, Query, Header import urllib.parse -from rdflib import Graph from enum import Enum -from nomad.config import config +from fastapi import Header, Query, Response +from rdflib import Graph +from nomad.config import config root_path = f'{config.services.api_base_path}/dcat' base_url = config.api_url(api='dcat') diff --git a/nomad/app/dcat/main.py b/nomad/app/dcat/main.py index 49d4d68d50fcfb197fb6d2afb15718fcf9414ce4..0fbfa0d4f6d193a255273a904c3afdca47f25769 100644 --- a/nomad/app/dcat/main.py +++ b/nomad/app/dcat/main.py @@ -16,10 +16,11 @@ # limitations under the License. # -from fastapi import FastAPI, status, Request +import traceback + +from fastapi import FastAPI, Request, status from fastapi.middleware.cors import CORSMiddleware from fastapi.responses import JSONResponse, RedirectResponse -import traceback from nomad import utils from nomad.config import config @@ -27,7 +28,6 @@ from nomad.config import config from .common import root_path from .routers import dcat - logger = utils.get_logger(__name__) diff --git a/nomad/app/dcat/mapping.py b/nomad/app/dcat/mapping.py index d22399a7628fba69adf3d0309b1b1cf1f63072dc..c15ec0a87693b66ae4be9c291f64ff839517dc42 100644 --- a/nomad/app/dcat/mapping.py +++ b/nomad/app/dcat/mapping.py @@ -16,15 +16,15 @@ # limitations under the License. # -from rdflib import Graph, Literal, RDF, URIRef, BNode -from rdflib.namespace import Namespace, DCAT, DCTERMS as DCT, FOAF, RDF +from rdflib import RDF, BNode, Graph, Literal, URIRef +from rdflib.namespace import DCAT, FOAF, RDF, Namespace +from rdflib.namespace import DCTERMS as DCT from nomad.config import config from nomad.datamodel import User from .common import url - VCARD = Namespace('http://www.w3.org/2006/vcard/ns#') HYDRA = Namespace('http://www.w3.org/ns/hydra/core#') diff --git a/nomad/app/dcat/routers/dcat.py b/nomad/app/dcat/routers/dcat.py index 99a2f0256a77339adb8efbef6ef2fcaafbdf2196..0aa5b6b6ece36af188994b1a6ac63f91d0b5ffb6 100644 --- a/nomad/app/dcat/routers/dcat.py +++ b/nomad/app/dcat/routers/dcat.py @@ -16,17 +16,17 @@ # limitations under the License. # +from datetime import date, datetime from enum import Enum -from fastapi import APIRouter, Query, Path, HTTPException, status, Depends -from datetime import datetime, date from elasticsearch_dsl import Q +from fastapi import APIRouter, Depends, HTTPException, Path, Query, status from nomad import utils -from nomad.utils import strip -from nomad.search import search -from nomad.app.v1.models import MetadataPagination, HTTPExceptionModel +from nomad.app.v1.models import HTTPExceptionModel, MetadataPagination from nomad.app.v1.utils import create_responses +from nomad.search import search +from nomad.utils import strip from ..common import rdf_response from ..mapping import Mapping diff --git a/nomad/app/h5grove_app.py b/nomad/app/h5grove_app.py index d409e01ee9ff2493a02a777427773bcc33a81d38..8e01225ff41914e31848e83f5e3d250ef4300f13 100644 --- a/nomad/app/h5grove_app.py +++ b/nomad/app/h5grove_app.py @@ -17,23 +17,24 @@ # from __future__ import annotations -from fastapi import FastAPI, status, Request, Depends -from fastapi.middleware.cors import CORSMiddleware -from fastapi.responses import JSONResponse -import traceback import re +import traceback import urllib.parse -import h5py -from typing import Any, IO from collections.abc import Callable +from typing import IO, Any -from h5grove import fastapi_utils as h5grove_router, utils as h5grove_utils +import h5py +from fastapi import Depends, FastAPI, Request, status +from fastapi.middleware.cors import CORSMiddleware +from fastapi.responses import JSONResponse +from h5grove import fastapi_utils as h5grove_router +from h5grove import utils as h5grove_utils from nomad import utils -from nomad.files import UploadFiles, PublicUploadFiles from nomad.app.v1.models import User from nomad.app.v1.routers.auth import create_user_dependency from nomad.app.v1.routers.uploads import get_upload_with_read_access +from nomad.files import PublicUploadFiles, UploadFiles logger = utils.get_logger(__name__) @@ -43,8 +44,9 @@ def open_zipped_h5_file( create_error: Callable[[int, str], Exception], h5py_options: dict[str, Any] = {}, ) -> h5py.File: - import re import io + import re + from nomad import files """ diff --git a/nomad/app/main.py b/nomad/app/main.py index 2168e5fd965f7e7bd939d5779e7f707dd2cf7c6f..29829f997ace12d293728e89518a0cb1c668850b 100644 --- a/nomad/app/main.py +++ b/nomad/app/main.py @@ -23,16 +23,17 @@ from fastapi import FastAPI, Response, status from fastapi.exception_handlers import ( http_exception_handler as default_http_exception_handler, ) -from starlette.exceptions import HTTPException as StarletteHTTPException from fastapi.responses import HTMLResponse, JSONResponse +from starlette.exceptions import HTTPException as StarletteHTTPException from starlette.middleware.base import BaseHTTPMiddleware from nomad import infrastructure from nomad.config import config from nomad.config.models.plugins import APIEntryPoint +from .static import GuiFiles +from .static import app as static_files_app from .v1.main import app as v1_app -from .static import app as static_files_app, GuiFiles class OasisAuthenticationMiddleware(BaseHTTPMiddleware): @@ -164,11 +165,10 @@ async def http_exception_handler(request, exc): @app.on_event('startup') async def startup_event(): - from nomad.cli.dev import get_gui_artifacts_js - from nomad.cli.dev import get_gui_config - from nomad.parsing.parsers import import_all_parsers from nomad import infrastructure + from nomad.cli.dev import get_gui_artifacts_js, get_gui_config from nomad.metainfo.elasticsearch_extension import entry_type + from nomad.parsing.parsers import import_all_parsers import_all_parsers() diff --git a/nomad/app/optimade/common.py b/nomad/app/optimade/common.py index fb0df1cc8513dbda9fea34702348aac99e47d50a..45824ef06bce9789286da9887d30f52151adbc1d 100644 --- a/nomad/app/optimade/common.py +++ b/nomad/app/optimade/common.py @@ -19,9 +19,8 @@ from typing import cast from nomad.metainfo.data_type import Datatype, to_optimade_type -from nomad.metainfo.metainfo import Quantity, Reference from nomad.metainfo.elasticsearch_extension import SearchQuantity, entry_type - +from nomad.metainfo.metainfo import Quantity, Reference _provider_specific_fields: dict[str, SearchQuantity] = None diff --git a/nomad/app/optimade/elasticsearch.py b/nomad/app/optimade/elasticsearch.py index d90607b3398c06e38654d7fd207da9c8c4c3a292..d966f7d7ccbba75f9e8c7b8c47d14695b43494a0 100644 --- a/nomad/app/optimade/elasticsearch.py +++ b/nomad/app/optimade/elasticsearch.py @@ -1,23 +1,23 @@ from typing import Any -from elasticsearch_dsl import Q +from elasticsearch_dsl import Q from optimade.filterparser import LarkParser +from optimade.models import StructureResource from optimade.server.entry_collections import EntryCollection from optimade.server.exceptions import BadRequest from optimade.server.mappers import StructureMapper from optimade.server.mappers.entries import classproperty -from optimade.models import StructureResource -from nomad.units import ureg -from nomad.atomutils import Formula -from nomad.search import search +from nomad import datamodel, files, utils from nomad.app.v1.models import MetadataPagination, MetadataRequired +from nomad.atomutils import Formula from nomad.config import config -from nomad import datamodel, files, utils +from nomad.search import search +from nomad.units import ureg -from .filterparser import _get_transformer as get_transformer -from .common import provider_specific_fields from ...archive import to_json +from .common import provider_specific_fields +from .filterparser import _get_transformer as get_transformer logger = utils.get_logger(__name__) diff --git a/nomad/app/optimade/filterparser.py b/nomad/app/optimade/filterparser.py index 6ff5b0e26abc54535c0acc9d4ba58625be5ce587..11caebaf9e9625d820b365292fc1b8ddaeb1a448 100644 --- a/nomad/app/optimade/filterparser.py +++ b/nomad/app/optimade/filterparser.py @@ -16,18 +16,16 @@ # limitations under the License. # -from elasticsearch_dsl import Q from cachetools import cached - +from elasticsearch_dsl import Q from optimade.filterparser import LarkParser +from optimade.filtertransformers.elasticsearch import ElasticsearchQuantity as Quantity from optimade.filtertransformers.elasticsearch import ( - ElasticsearchQuantity as Quantity, ElasticTransformer as OPTElasticTransformer, ) from .common import provider_specific_fields - _parser = LarkParser(version=(1, 0, 1)) diff --git a/nomad/app/resources/common.py b/nomad/app/resources/common.py index f0cedfb72be356e1a06565929ef2d59554392ec2..96a0e9c977d4e6b25fb515c2b45d029f5563b760 100644 --- a/nomad/app/resources/common.py +++ b/nomad/app/resources/common.py @@ -18,6 +18,5 @@ from nomad.config import config - root_path = f'{config.services.api_base_path}/resources' base_url = config.api_url(api='resources') diff --git a/nomad/app/resources/main.py b/nomad/app/resources/main.py index e5c4b6992de2ffdd2ea96610a917f511023567f9..ab6e17d11d972638ef073fb55525c9156328d475 100644 --- a/nomad/app/resources/main.py +++ b/nomad/app/resources/main.py @@ -16,18 +16,18 @@ # limitations under the License. # -from fastapi import FastAPI, status, Request -from fastapi.middleware.cors import CORSMiddleware -from fastapi.responses import JSONResponse import traceback + from celery.signals import worker_process_init +from fastapi import FastAPI, Request, status +from fastapi.middleware.cors import CORSMiddleware +from fastapi.responses import JSONResponse from nomad import utils from nomad.config import config from .routers import resources - logger = utils.get_logger(__name__) mongo_client_resources = None diff --git a/nomad/app/resources/routers/resources.py b/nomad/app/resources/routers/resources.py index 242e82950d2af0bfe5507004d665ce1f1e8e093e..45f4f9559bd3e8a20bc373aca798d13b4d6eb6d4 100644 --- a/nomad/app/resources/routers/resources.py +++ b/nomad/app/resources/routers/resources.py @@ -16,35 +16,36 @@ # limitations under the License. # -import re -import os -import io -import bs4 import asyncio -import httpx +import io +import os +import re +from datetime import datetime from enum import Enum -from fastapi import APIRouter, Query as FastApiQuery -from pydantic import BaseModel, Field from typing import Any -from datetime import datetime + import ase.io +import bs4 +import httpx +from asgiref.sync import async_to_sync +from fastapi import APIRouter +from fastapi import Query as FastApiQuery from mongoengine import ( - Document, - StringField, + BooleanField, DateTimeField, + Document, IntField, ListField, - BooleanField, + StringField, ) from mongoengine.queryset.visitor import Q -from asgiref.sync import async_to_sync +from pydantic import BaseModel, Field from nomad import utils -from nomad.config import config from nomad.atomutils import Formula +from nomad.config import config from nomad.processing.base import app - logger = utils.get_logger(__name__) router = APIRouter() diff --git a/nomad/app/v1/main.py b/nomad/app/v1/main.py index 2c3e7b66062d037e8f0d3f26283ec3515d423317..af4def92478e5e50ecd01f38c3e3fc652838063c 100644 --- a/nomad/app/v1/main.py +++ b/nomad/app/v1/main.py @@ -18,9 +18,9 @@ import traceback -from fastapi import FastAPI, status, Request +from fastapi import FastAPI, Request, status from fastapi.middleware.cors import CORSMiddleware -from fastapi.responses import JSONResponse, RedirectResponse, ORJSONResponse +from fastapi.responses import JSONResponse, ORJSONResponse, RedirectResponse from pyinstrument import Profiler from starlette.middleware import Middleware from starlette.middleware.base import BaseHTTPMiddleware @@ -29,22 +29,23 @@ from starlette.types import ASGIApp, Receive, Scope, Send from nomad import utils from nomad.config import config + from .common import root_path from .routers import ( - users, - entries, - materials, auth, - info, datasets, - uploads, - suggestions, - metainfo, - north, - systems, + entries, federation, graph, groups, + info, + materials, + metainfo, + north, + suggestions, + systems, + uploads, + users, ) logger = utils.get_logger(__name__) diff --git a/nomad/app/v1/models/graph/graph_models.py b/nomad/app/v1/models/graph/graph_models.py index 480221c93f4a6393027888b2a89eafc47925717c..dff9d7d42bdf3fc109d4d23e1a32c92613bccf00 100644 --- a/nomad/app/v1/models/graph/graph_models.py +++ b/nomad/app/v1/models/graph/graph_models.py @@ -17,35 +17,37 @@ # from __future__ import annotations + from typing import Any, Literal -from pydantic import BaseModel, ConfigDict, Field, Extra -from ..groups import UserGroup, UserGroupPagination, UserGroupQuery +from pydantic import BaseModel, ConfigDict, Extra, Field -from nomad.graph.model import ( - RequestConfig, - DatasetQuery, - MetainfoQuery, - MetainfoPagination, +from nomad.app.v1.models.graph.utils import ( + generate_request_model, + generate_response_model, + mapped, ) -from nomad.metainfo.pydantic_extension import PydanticModel -from nomad.datamodel.data import User as UserModel from nomad.app.v1.models.models import Metadata, MetadataResponse -from nomad.app.v1.routers.datasets import Dataset as DatasetV1, DatasetPagination +from nomad.app.v1.routers.datasets import Dataset as DatasetV1 +from nomad.app.v1.routers.datasets import DatasetPagination from nomad.app.v1.routers.uploads import ( + EntryProcData, + EntryProcDataPagination, + PaginationResponse, UploadProcData, UploadProcDataPagination, UploadProcDataQuery, - PaginationResponse, - EntryProcData, - EntryProcDataPagination, ) - -from nomad.app.v1.models.graph.utils import ( - generate_request_model, - generate_response_model, - mapped, +from nomad.datamodel.data import User as UserModel +from nomad.graph.model import ( + DatasetQuery, + MetainfoPagination, + MetainfoQuery, + RequestConfig, ) +from nomad.metainfo.pydantic_extension import PydanticModel + +from ..groups import UserGroup, UserGroupPagination, UserGroupQuery class Error(BaseModel): diff --git a/nomad/app/v1/models/graph/utils.py b/nomad/app/v1/models/graph/utils.py index 6d420a6e1002c51bca54bd55ccbb815ddac4c34d..6c871c55478ef09754f5998ac8aa97686b1e645f 100644 --- a/nomad/app/v1/models/graph/utils.py +++ b/nomad/app/v1/models/graph/utils.py @@ -17,32 +17,33 @@ # from __future__ import annotations + +import sys +from collections.abc import Callable +from datetime import datetime +from types import UnionType from typing import ( - Optional, - Literal, - Union, Any, ForwardRef, - get_type_hints, - get_origin, - get_args, + Literal, + Optional, + Union, cast, + get_args, + get_origin, + get_type_hints, ) -from collections.abc import Callable -from types import UnionType -from datetime import datetime + from pydantic import ( BaseModel, ConfigDict, + Field, TypeAdapter, + ValidationError, create_model, - Field, model_validator, - ValidationError, ) from pydantic.config import ConfigDict as BaseConfigDict -import sys - ref_prefix = '#/components/schemas' request_suffix = 'Request' diff --git a/nomad/app/v1/models/groups.py b/nomad/app/v1/models/groups.py index 3a83a81818de492b89788d5c500a011d2895981d..dbdae164fba850f19314e7cb855e8312bf9d10d3 100644 --- a/nomad/app/v1/models/groups.py +++ b/nomad/app/v1/models/groups.py @@ -1,9 +1,4 @@ -from pydantic import ( - BaseModel, - ConfigDict, - Field, - field_validator, -) +from pydantic import BaseModel, ConfigDict, Field, field_validator from pydantic_core import PydanticCustomError from .pagination import Direction, Pagination, PaginationResponse diff --git a/nomad/app/v1/models/models.py b/nomad/app/v1/models/models.py index 32841a883274651dca779aae60bfe2c262b7e397..7dab17d0f8e71bbe934b4646085151a71b885b2e 100644 --- a/nomad/app/v1/models/models.py +++ b/nomad/app/v1/models/models.py @@ -16,25 +16,25 @@ # limitations under the License. # import datetime -from enum import Enum import fnmatch import json import re -from typing import Any from collections.abc import Mapping +from enum import Enum +from typing import Annotated, Any from fastapi import Body, HTTPException, Request from fastapi import Query as FastApiQuery from pydantic import ( # noqa: F401 - field_validator, - model_validator, - StringConstraints, - ConfigDict, BaseModel, + ConfigDict, Field, StrictBool, StrictFloat, StrictInt, + StringConstraints, + field_validator, + model_validator, ) from pydantic.main import create_model from pydantic_core import PydanticCustomError @@ -49,8 +49,6 @@ from nomad.metainfo.elasticsearch_extension import ( from nomad.utils import strip from .pagination import Pagination, PaginationResponse -from typing import Annotated - User: Any = datamodel.User.m_def.a_pydantic.model # It is important that datetime.datetime comes last. Otherwise, number valued strings diff --git a/nomad/app/v1/models/pagination.py b/nomad/app/v1/models/pagination.py index 6cb92b5c54ba81f45d720f5afffe2914fe2d18cb..c3fba17369172950363d5af640d5c8390cadb97c 100644 --- a/nomad/app/v1/models/pagination.py +++ b/nomad/app/v1/models/pagination.py @@ -1,12 +1,7 @@ from enum import Enum + from fastapi import HTTPException, Request -from pydantic import ( - BaseModel, - ConfigDict, - Field, - field_validator, - model_validator, -) +from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator from pydantic_core import PydanticCustomError from nomad.app.v1.utils import update_url_query_arguments diff --git a/nomad/app/v1/routers/auth.py b/nomad/app/v1/routers/auth.py index d64981f471a270e93d84e243b94e695cc198ae68..01740cda39d9ec2026a2d50fd7694631716c9052 100644 --- a/nomad/app/v1/routers/auth.py +++ b/nomad/app/v1/routers/auth.py @@ -16,34 +16,29 @@ # limitations under the License. # -import hmac +import datetime import hashlib +import hmac import uuid -import requests -from enum import Enum -from typing import cast from collections.abc import Callable -from inspect import Parameter, signature +from enum import Enum from functools import wraps -from fastapi import ( - APIRouter, - Depends, - Query as FastApiQuery, - Request, - HTTPException, - status, -) +from inspect import Parameter, signature +from typing import cast + +import jwt +import requests +from fastapi import APIRouter, Depends, HTTPException, Request, status +from fastapi import Query as FastApiQuery from fastapi.security import OAuth2PasswordBearer, OAuth2PasswordRequestForm from pydantic import BaseModel -import jwt -import datetime -from nomad import utils, infrastructure, datamodel +from nomad import datamodel, infrastructure, utils from nomad.config import config from nomad.utils import get_logger, strip from ..common import root_path -from ..models import User, HTTPExceptionModel +from ..models import HTTPExceptionModel, User from ..utils import create_responses logger = get_logger(__name__) diff --git a/nomad/app/v1/routers/datasets.py b/nomad/app/v1/routers/datasets.py index 34eec597479dcf5f8d1a2cdd3eafe137f43f03db..6038f44f98ccd1af3a4174e7022a3161e0b72d2c 100644 --- a/nomad/app/v1/routers/datasets.py +++ b/nomad/app/v1/routers/datasets.py @@ -17,44 +17,37 @@ # import re +from datetime import datetime from enum import Enum from typing import cast -from fastapi import ( - APIRouter, - Request, - Depends, - Query as FastApiQuery, - Path, - HTTPException, - status, -) -from pydantic import field_validator, BaseModel, Field -from datetime import datetime -from nomad import utils, datamodel, processing +from fastapi import APIRouter, Depends, HTTPException, Path, Request, status +from fastapi import Query as FastApiQuery +from pydantic import BaseModel, Field, field_validator + +from nomad import datamodel, processing, utils from nomad.config import config -from nomad.metainfo.elasticsearch_extension import entry_type -from nomad.utils import strip, create_uuid from nomad.datamodel import Dataset as DatasetDefinitionCls from nomad.doi import DOI, DOIException +from nomad.metainfo.elasticsearch_extension import entry_type from nomad.search import search, update_by_query +from nomad.utils import create_uuid, strip -from .auth import create_user_dependency -from .entries import _do_exhaustive_search -from ..utils import create_responses, parameter_dependency_from_model from ..models import ( + Any_, + Direction, + HTTPExceptionModel, + MetadataPagination, + MetadataRequired, + Owner, Pagination, PaginationResponse, - MetadataPagination, Query, - HTTPExceptionModel, User, - Direction, - Owner, - Any_, - MetadataRequired, ) - +from ..utils import create_responses, parameter_dependency_from_model +from .auth import create_user_dependency +from .entries import _do_exhaustive_search router = APIRouter() diff --git a/nomad/app/v1/routers/entries.py b/nomad/app/v1/routers/entries.py index 7f25c97581bc3b2e33bf94f4c50e2f230a3906c1..f974e154214bba6607ac9a7d8f79ff0a7267ab12 100644 --- a/nomad/app/v1/routers/entries.py +++ b/nomad/app/v1/routers/entries.py @@ -15,89 +15,75 @@ # See the License for the specific language governing permissions and # limitations under the License. # +import io +import json +import os.path +from collections.abc import Iterator from datetime import datetime - from enum import Enum from typing import Any -from collections.abc import Iterator -from fastapi import ( - APIRouter, - Depends, - Path, - status, - HTTPException, - Request, - Query as QueryParameter, - Body, -) -from fastapi.responses import StreamingResponse, ORJSONResponse -from fastapi.exceptions import RequestValidationError -from pydantic import ( - ConfigDict, - field_validator, - BaseModel, - Field, -) -import os.path -import io -import json + import orjson +import yaml +from fastapi import APIRouter, Body, Depends, HTTPException, Path, Request, status +from fastapi import Query as QueryParameter +from fastapi.exceptions import RequestValidationError +from fastapi.responses import ORJSONResponse, StreamingResponse +from pydantic import BaseModel, ConfigDict, Field, field_validator from pydantic.main import create_model from starlette.responses import Response -import yaml -from nomad import files, utils, metainfo, processing as proc -from nomad import datamodel +from nomad import datamodel, files, metainfo, utils +from nomad import processing as proc +from nomad.archive import ArchiveQueryError, RequiredReader, RequiredValidationError from nomad.config import config from nomad.config.models.config import Reprocess from nomad.datamodel import EditableUserMetadata from nomad.datamodel.context import ServerContext from nomad.files import StreamedFile, create_zipstream_async -from nomad.processing.data import Upload -from nomad.utils import strip -from nomad.archive import RequiredReader, RequiredValidationError, ArchiveQueryError from nomad.groups import get_group_ids +from nomad.metainfo.elasticsearch_extension import entry_type +from nomad.processing.data import Upload from nomad.search import ( AuthenticationRequiredError, QueryValidationError, SearchError, search, - update_metadata as es_update_metadata, ) -from nomad.metainfo.elasticsearch_extension import entry_type +from nomad.search import update_metadata as es_update_metadata +from nomad.utils import strip -from .auth import create_user_dependency -from ..utils import ( - create_download_stream_zipped, - create_download_stream_raw_file, - browser_download_headers, - DownloadItem, - create_responses, - log_query, -) from ..models import ( Aggregation, - Pagination, - PaginationResponse, + Files, + HTTPExceptionModel, + Metadata, + MetadataEditRequest, MetadataPagination, - TermsAggregation, - WithQuery, - WithQueryAndPagination, MetadataRequired, MetadataResponse, - Metadata, - MetadataEditRequest, - Files, - Query, - User, Owner, + Pagination, + PaginationResponse, + Query, QueryParameters, - metadata_required_parameters, + TermsAggregation, + User, + WithQuery, + WithQueryAndPagination, files_parameters, metadata_pagination_parameters, - HTTPExceptionModel, + metadata_required_parameters, ) - +from ..utils import ( + DownloadItem, + browser_download_headers, + create_download_stream_raw_file, + create_download_stream_zipped, + create_responses, + log_query, +) +from .auth import create_user_dependency router = APIRouter() diff --git a/nomad/app/v1/routers/federation.py b/nomad/app/v1/routers/federation.py index 11c7ff4eb5c570e0067209df0cc37967ff896a8f..c554228d0bffcc7b92666bed3a3f82dbcc7a4329 100644 --- a/nomad/app/v1/routers/federation.py +++ b/nomad/app/v1/routers/federation.py @@ -24,7 +24,7 @@ import socket import zlib from enum import Enum -from fastapi import Request, HTTPException +from fastapi import HTTPException, Request from fastapi.routing import APIRouter from nomad import utils diff --git a/nomad/app/v1/routers/graph.py b/nomad/app/v1/routers/graph.py index c3715787c69d2c448f8acca55777eefc5eff2a50..1d1006c53c11f2f8198d23cda71807b6c1003ee8 100644 --- a/nomad/app/v1/routers/graph.py +++ b/nomad/app/v1/routers/graph.py @@ -18,21 +18,22 @@ from enum import Enum -from fastapi import Depends, APIRouter, Body, HTTPException +from fastapi import APIRouter, Body, Depends, HTTPException from fastapi.responses import ORJSONResponse +from nomad.app.v1.models.graph import GraphRequest, GraphResponse from nomad.graph.graph_reader import ( - MongoReader, ConfigError, GeneralReader, - UserReader, + MongoReader, Token, + UserReader, ) from nomad.graph.lazy_wrapper import LazyWrapper + +from ..models import User from .auth import create_user_dependency from .entries import EntriesArchive -from ..models import User -from nomad.app.v1.models.graph import GraphRequest, GraphResponse router = APIRouter() diff --git a/nomad/app/v1/routers/groups.py b/nomad/app/v1/routers/groups.py index a41342eb461808d5de715b06b921271fa0d231e2..945b95f75706b66025c04ac8970047ae2b48e5bd 100644 --- a/nomad/app/v1/routers/groups.py +++ b/nomad/app/v1/routers/groups.py @@ -29,9 +29,6 @@ from nomad.app.v1.models.groups import ( ) from nomad.app.v1.models.pagination import PaginationResponse from nomad.app.v1.utils import parameter_dependency_from_model - -from fastapi import APIRouter, Depends, HTTPException, status - from nomad.datamodel import User as UserDataModel from nomad.groups import MongoUserGroup from nomad.groups import create_user_group as create_mongo_user_group diff --git a/nomad/app/v1/routers/info.py b/nomad/app/v1/routers/info.py index 1efa9a15fcbd3100f58a80725d97c49079bc7b1b..0e63c5778822a8f4cca30a14645ac58435b09d2a 100644 --- a/nomad/app/v1/routers/info.py +++ b/nomad/app/v1/routers/info.py @@ -20,22 +20,22 @@ API endpoint that deliver backend configuration details. """ +from datetime import datetime from enum import Enum from typing import Any -from datetime import datetime + from fastapi.routing import APIRouter from pydantic.fields import Field from pydantic.main import BaseModel from nomad import normalizing +from nomad.app.v1.models import Aggregation, StatisticsAggregation from nomad.config import config -from nomad.utils import strip -from nomad.search import search +from nomad.metainfo.elasticsearch_extension import entry_type from nomad.parsing import parsers from nomad.parsing.parsers import code_metadata -from nomad.app.v1.models import Aggregation, StatisticsAggregation -from nomad.metainfo.elasticsearch_extension import entry_type - +from nomad.search import search +from nomad.utils import strip router = APIRouter() diff --git a/nomad/app/v1/routers/materials.py b/nomad/app/v1/routers/materials.py index 579342c5fb1e8e0ec7a6fd89f6642fc8c562300c..70c6ef56b6b52531df4ac31d5a9090e82abcfeb3 100644 --- a/nomad/app/v1/routers/materials.py +++ b/nomad/app/v1/routers/materials.py @@ -17,32 +17,36 @@ # from typing import Any -from fastapi import APIRouter, Depends, Path, status, HTTPException, Request + +from fastapi import APIRouter, Depends, HTTPException, Path, Request, status from fastapi.exception_handlers import RequestValidationError from pydantic import BaseModel, Field from nomad import utils +from nomad.metainfo.elasticsearch_extension import material_index, material_type +from nomad.search import ( + AuthenticationRequiredError, + QueryValidationError, + SearchError, + search, +) from nomad.utils import strip -from nomad.search import AuthenticationRequiredError, SearchError -from nomad.search import search, QueryValidationError -from nomad.metainfo.elasticsearch_extension import material_type, material_index -from .auth import create_user_dependency -from ..utils import create_responses from ..models import ( - User, - Owner, - WithQuery, - MetadataResponse, + HTTPExceptionModel, Metadata, MetadataPagination, MetadataRequired, + MetadataResponse, + Owner, + QueryParameters, + User, + WithQuery, metadata_pagination_parameters, metadata_required_parameters, - QueryParameters, - HTTPExceptionModel, ) - +from ..utils import create_responses +from .auth import create_user_dependency router = APIRouter() diff --git a/nomad/app/v1/routers/metainfo.py b/nomad/app/v1/routers/metainfo.py index 882e21b6d283ad168ff593079f6ef5a57e84af98..a92fccbb529875004a86ca8dc889a7ab047dea92 100644 --- a/nomad/app/v1/routers/metainfo.py +++ b/nomad/app/v1/routers/metainfo.py @@ -19,16 +19,16 @@ import copy import datetime from typing import Any -from fastapi import APIRouter, Path, status, HTTPException +from fastapi import APIRouter, HTTPException, Path, status from pydantic import BaseModel, Field from nomad.app.v1.models import HTTPExceptionModel from nomad.app.v1.utils import create_responses -from nomad.metainfo import Package -from nomad.metainfo.metainfo import MSection, Section, Quantity, Datetime, JSON -from nomad.metainfo.mongoengine_extension import MongoDocument, Mongo -from nomad.utils import strip, get_logger from nomad.config import config +from nomad.metainfo import Package +from nomad.metainfo.metainfo import JSON, Datetime, MSection, Quantity, Section +from nomad.metainfo.mongoengine_extension import Mongo, MongoDocument +from nomad.utils import get_logger, strip logger = get_logger(__name__) diff --git a/nomad/app/v1/routers/north.py b/nomad/app/v1/routers/north.py index c026348e6ed29d9f52a71beb52d5badb37b00fe1..73c795bcf98baf1a2e78a37a02c94c2ccfe14312 100644 --- a/nomad/app/v1/routers/north.py +++ b/nomad/app/v1/routers/north.py @@ -17,23 +17,23 @@ # import os -import requests - from enum import Enum -from nomad.groups import get_group_ids -from pydantic import BaseModel -from fastapi import APIRouter, Depends, status, HTTPException + +import requests +from fastapi import APIRouter, Depends, HTTPException, status from mongoengine.queryset.visitor import Q +from pydantic import BaseModel +from nomad.app.v1.routers.auth import generate_simple_token from nomad.config import config from nomad.config.models.north import NORTHTool -from nomad.utils import strip, get_logger, slugify +from nomad.groups import get_group_ids from nomad.processing import Upload -from nomad.app.v1.routers.auth import generate_simple_token -from .auth import create_user_dependency -from ..models import User, HTTPExceptionModel -from ..utils import create_responses +from nomad.utils import get_logger, slugify, strip +from ..models import HTTPExceptionModel, User +from ..utils import create_responses +from .auth import create_user_dependency TOOLS = {k: v for k, v in config.north.tools.filtered_items()} diff --git a/nomad/app/v1/routers/suggestions.py b/nomad/app/v1/routers/suggestions.py index 0d0b68229f4edbc30b66376152d07338580257ed..2bd6e515e3a2d446c29cde595e9bd5bb126f8195 100644 --- a/nomad/app/v1/routers/suggestions.py +++ b/nomad/app/v1/routers/suggestions.py @@ -17,17 +17,17 @@ # from collections import defaultdict -from pydantic import BaseModel, Field -from fastapi import APIRouter, Depends, Request, HTTPException, status + +from elasticsearch.exceptions import RequestError from elasticsearch_dsl import Search from elasticsearch_dsl.utils import AttrList -from elasticsearch.exceptions import RequestError +from fastapi import APIRouter, Depends, HTTPException, Request, status +from pydantic import BaseModel, Field from nomad.metainfo.elasticsearch_extension import entry_index, entry_type -from .auth import create_user_dependency from ..models import User - +from .auth import create_user_dependency router = APIRouter() diff --git a/nomad/app/v1/routers/systems.py b/nomad/app/v1/routers/systems.py index 6d56d61d2c1b3763c1eaf51cd93b8b35ef5227fc..c92e1c44f4520b1198a395817a214afec987ccc9 100644 --- a/nomad/app/v1/routers/systems.py +++ b/nomad/app/v1/routers/systems.py @@ -15,29 +15,27 @@ # See the License for the specific language governing permissions and # limitations under the License. # -from io import StringIO, BytesIO import sys from collections import OrderedDict from enum import Enum +from io import BytesIO, StringIO +import ase.build +import ase.io import numpy as np -from fastapi import APIRouter, Depends, Path, Query, status, HTTPException +from fastapi import APIRouter, Depends, HTTPException, Path, Query, status from fastapi.responses import Response -import ase.io -import ase.build -from nomad.units import ureg -from nomad.utils import strip, deep_get, query_list_to_dict -from nomad.atomutils import Formula, wrap_positions, unwrap_positions -from nomad.normalizing.common import ( - ase_atoms_from_nomad_atoms, -) +from nomad.atomutils import Formula, unwrap_positions, wrap_positions from nomad.datamodel.metainfo.system import Atoms as NOMADAtoms -from .entries import answer_entry_archive_request +from nomad.normalizing.common import ase_atoms_from_nomad_atoms +from nomad.units import ureg +from nomad.utils import deep_get, query_list_to_dict, strip -from .auth import create_user_dependency +from ..models import HTTPExceptionModel, User from ..utils import create_responses -from ..models import User, HTTPExceptionModel +from .auth import create_user_dependency +from .entries import answer_entry_archive_request router = APIRouter() diff --git a/nomad/app/v1/routers/uploads.py b/nomad/app/v1/routers/uploads.py index 5c5400701493cfdc1cea8e4e40e67a1b7f5d4d8b..fa6ff235ca0c7454634a8c64fe93f062507048f1 100644 --- a/nomad/app/v1/routers/uploads.py +++ b/nomad/app/v1/routers/uploads.py @@ -15,92 +15,79 @@ # See the License for the specific language governing permissions and # limitations under the License. # -import os import io +import os import shutil -import zipfile import tarfile -from enum import Enum +import zipfile from datetime import datetime +from enum import Enum from typing import Any, cast -from pydantic import ( - field_validator, - ConfigDict, - BaseModel, - Field, - model_validator, -) -from mongoengine.queryset.visitor import Q from urllib.parse import unquote + from fastapi import ( APIRouter, - Request, + Body, + Depends, File, + HTTPException, + Path, + Request, UploadFile, status, - Depends, - Body, - Path, - Query as FastApiQuery, - HTTPException, ) -from fastapi.responses import StreamingResponse, FileResponse +from fastapi import Query as FastApiQuery from fastapi.exceptions import RequestValidationError +from fastapi.responses import FileResponse, StreamingResponse +from mongoengine.queryset.visitor import Q +from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator from pydantic_core import PydanticCustomError -from nomad import utils, files -from nomad.common import is_safe_relative_path, is_safe_basename -from nomad.config import config -from nomad.config.models.plugins import ExampleUploadEntryPoint -from nomad.files import ( - StagingUploadFiles, - PublicUploadFiles, -) +from nomad import files, utils from nomad.bundles import BundleExporter, BundleImporter +from nomad.common import get_compression_format, is_safe_basename, is_safe_relative_path +from nomad.config import config from nomad.config.models.config import Reprocess +from nomad.config.models.plugins import ExampleUploadEntryPoint +from nomad.files import PublicUploadFiles, StagingUploadFiles from nomad.groups import get_group_ids from nomad.processing import ( - Upload, Entry, + MetadataEditRequestHandler, ProcessAlreadyRunning, ProcessStatus, - MetadataEditRequestHandler, + Upload, ) -from nomad.common import get_compression_format +from nomad.search import QueryValidationError, search, search_iterator +from nomad.search import refresh as search_refresh from nomad.utils import strip -from nomad.search import ( - search, - search_iterator, - refresh as search_refresh, - QueryValidationError, -) -from .auth import create_user_dependency, generate_upload_token from ..models import ( - MetadataPagination, - User, Direction, - Pagination, - PaginationResponse, - HTTPExceptionModel, Files, - files_parameters, + HTTPExceptionModel, + MetadataEditRequest, + MetadataPagination, + MetadataRequired, Owner, + Pagination, + PaginationResponse, + User, WithQuery, - MetadataRequired, - MetadataEditRequest, + files_parameters, restrict_query_to_upload, ) -from .entries import EntryArchiveResponse, answer_entry_archive_request from ..utils import ( - parameter_dependency_from_model, - create_responses, DownloadItem, browser_download_headers, - create_download_stream_zipped, create_download_stream_raw_file, + create_download_stream_zipped, + create_responses, create_stream_from_string, + parameter_dependency_from_model, ) +from .auth import create_user_dependency, generate_upload_token +from .entries import EntryArchiveResponse, answer_entry_archive_request router = APIRouter() diff --git a/nomad/app/v1/routers/users.py b/nomad/app/v1/routers/users.py index 50c003e8068b22f3b1cdb1b045e07bce7096396b..3e48ee899559efa081d41d6c16481e6ea2200028 100644 --- a/nomad/app/v1/routers/users.py +++ b/nomad/app/v1/routers/users.py @@ -18,16 +18,16 @@ from enum import Enum -from fastapi import Depends, APIRouter, status, HTTPException, Query +from fastapi import APIRouter, Depends, HTTPException, Query, status from pydantic.main import BaseModel -from nomad import infrastructure, datamodel +from nomad import datamodel, infrastructure from nomad.config import config from nomad.utils import strip -from .auth import create_user_dependency -from ..models import User, HTTPExceptionModel +from ..models import HTTPExceptionModel, User from ..utils import create_responses +from .auth import create_user_dependency router = APIRouter() diff --git a/nomad/app/v1/utils.py b/nomad/app/v1/utils.py index 1a6b04574c8270b3d826be958f6fe98d15569ba7..c62316fe4e8a02263d5a69014c25d6ea913939d7 100644 --- a/nomad/app/v1/utils.py +++ b/nomad/app/v1/utils.py @@ -16,19 +16,21 @@ # limitations under the License. # -from typing import Any -from collections.abc import Iterator -from types import FunctionType -import urllib +import gzip +import inspect import io import json -import os -import inspect -from fastapi import Request, Query, HTTPException, status # noqa: F401 -from pydantic import ValidationError, BaseModel # noqa: F401 -import gzip import lzma -from nomad.files import UploadFiles, StreamedFile, create_zipstream +import os +import urllib +from collections.abc import Iterator +from types import FunctionType +from typing import Any + +from fastapi import HTTPException, Query, Request, status # noqa: F401 +from pydantic import BaseModel, ValidationError # noqa: F401 + +from nomad.files import StreamedFile, UploadFiles, create_zipstream def parameter_dependency_from_model( diff --git a/nomad/archive/converter.py b/nomad/archive/converter.py index b72233f3ffdf3053471751b669d3cd11ca181780..96b6e494fd78e1b69613958571f556b81c4273ef 100644 --- a/nomad/archive/converter.py +++ b/nomad/archive/converter.py @@ -21,15 +21,14 @@ import functools import hashlib import os.path import signal +from collections.abc import Callable, Iterable from concurrent.futures import ProcessPoolExecutor from multiprocessing import Manager -from collections.abc import Callable -from collections.abc import Iterable -from nomad.config import config -from nomad.archive import to_json, read_archive +from nomad.archive import read_archive, to_json from nomad.archive.storage_v2 import ArchiveWriter as ArchiveWriterNew -from nomad.files import StagingUploadFiles, PublicUploadFiles +from nomad.config import config +from nomad.files import PublicUploadFiles, StagingUploadFiles from nomad.infrastructure import setup from nomad.processing import Upload diff --git a/nomad/archive/partial.py b/nomad/archive/partial.py index 0f503dbd06d426b106c1f3e67c26531e2aabb138..6d0f4012f0096975be51eb4ab53d6596a5b4436b 100644 --- a/nomad/archive/partial.py +++ b/nomad/archive/partial.py @@ -20,16 +20,16 @@ from typing import Any from nomad import infrastructure from nomad.config import config +from nomad.datamodel import EntryArchive +from nomad.datamodel.metainfo.common import FastAccess from nomad.metainfo import ( - MSection, Definition, + MSection, Quantity, Reference, - SubSection, Section, + SubSection, ) -from nomad.datamodel import EntryArchive -from nomad.datamodel.metainfo.common import FastAccess def create_partial_archive(archive: EntryArchive) -> dict: diff --git a/nomad/archive/query.py b/nomad/archive/query.py index 2097a2f7ba8a4381e6a42c1ad00c2ff5bcb428eb..09696fcdbcb95ce81219b57fe8511f4e984ce78a 100644 --- a/nomad/archive/query.py +++ b/nomad/archive/query.py @@ -18,13 +18,13 @@ import functools import re -from typing import Any from collections.abc import Callable from io import BytesIO +from typing import Any from nomad import utils -from .storage import ArchiveReader, ArchiveList, ArchiveDict, to_json, read_archive +from .storage import ArchiveDict, ArchiveList, ArchiveReader, read_archive, to_json _query_archive_key_pattern = re.compile(r'^([\s\w\-]+)(\[([-?0-9]*)(:([-?0-9]*))?])?$') diff --git a/nomad/archive/required.py b/nomad/archive/required.py index df5fa96f51311e0f25c34be7dfb6b355842689cf..402e116ad6fdaea53b8cfb08ae25137145984c31 100644 --- a/nomad/archive/required.py +++ b/nomad/archive/required.py @@ -28,24 +28,25 @@ from fastapi import HTTPException from nomad import utils from nomad.metainfo import ( Definition, - Section, + Package, Quantity, - SubSection, - Reference, QuantityReference, + Reference, + Section, SectionReference, - Package, + SubSection, ) + +from ..datamodel.context import ServerContext, parse_path from .query import ( ArchiveQueryError, - to_json, - _query_archive_key_pattern, - _extract_key_and_index, _extract_child, + _extract_key_and_index, + _query_archive_key_pattern, + to_json, ) -from .storage import ArchiveReader, ArchiveList, ArchiveError, ArchiveDict +from .storage import ArchiveDict, ArchiveError, ArchiveList, ArchiveReader from .storage_v2 import ArchiveDict as NewArchiveDict -from ..datamodel.context import parse_path, ServerContext class RequiredValidationError(Exception): diff --git a/nomad/archive/storage.py b/nomad/archive/storage.py index fa90306564d748f958e1c5d6695a6f650909a50c..29102e1eae6ddd68cf10542a797954c4804d41f2 100644 --- a/nomad/archive/storage.py +++ b/nomad/archive/storage.py @@ -17,12 +17,10 @@ # from __future__ import annotations -from typing import Any, cast -from collections.abc import Generator -from io import BytesIO, BufferedReader -from collections.abc import Mapping, Sequence - import struct +from collections.abc import Generator, Mapping, Sequence +from io import BufferedReader, BytesIO +from typing import Any, cast import msgspec @@ -328,10 +326,8 @@ def read_archive(file_or_path: str | BytesIO, **kwargs) -> ArchiveReader: will lazily load data as it is used. The mapping needs to be closed or used within a 'with' statement to free the underlying file resource after use. """ - from .storage_v2 import ( - ArchiveWriter as ArchiveWriterNew, - ArchiveReader as ArchiveReaderNew, - ) + from .storage_v2 import ArchiveReader as ArchiveReaderNew + from .storage_v2 import ArchiveWriter as ArchiveWriterNew # todo: replace implementation to enable automatic conversion # if isinstance(file_or_path, str): diff --git a/nomad/archive/storage_v2.py b/nomad/archive/storage_v2.py index 6a5d1e59af5aa7ad6dd82a3b87d647c44927df1c..ffb8fdaf6fdfdc0eca76797f591756a71ea0934a 100644 --- a/nomad/archive/storage_v2.py +++ b/nomad/archive/storage_v2.py @@ -27,8 +27,8 @@ from bitarray import bitarray from msgpack import Unpacker from nomad import utils -from nomad.config import config from nomad.archive import ArchiveError +from nomad.config import config _packer = msgpack.Packer(autoreset=True, use_bin_type=True) diff --git a/nomad/atomutils.py b/nomad/atomutils.py index 3b7a6f5cc14e4cf499fc04f6e30b74362297ad6b..d0deb7ded1bd8e66ea46a6b4bb693d7ad44391b1 100644 --- a/nomad/atomutils.py +++ b/nomad/atomutils.py @@ -23,14 +23,10 @@ import itertools import logging import math import re +from collections.abc import Iterable from functools import reduce from string import ascii_uppercase -from typing import ( - TYPE_CHECKING, - Any, - cast, -) -from collections.abc import Iterable +from typing import TYPE_CHECKING, Any, cast import ase.data import ase.geometry diff --git a/nomad/bundles.py b/nomad/bundles.py index 9ed5ef631607eee8d0b02f2fb0180c3edc2e92b0..93c45612e1fa0d47e75ed604e9ae4406517a5794 100644 --- a/nomad/bundles.py +++ b/nomad/bundles.py @@ -8,35 +8,36 @@ # See the License for the specific language governing permissions and # limitations under the License. # -from typing import cast, Any -from collections.abc import Iterable -import os import json +import os +from collections.abc import Iterable from datetime import datetime, timedelta +from typing import Any, cast + +from fastapi import HTTPException, status from packaging import version -from nomad import utils, datamodel, search +from nomad import datamodel, search, utils from nomad.config import config -from nomad.config.models.config import BundleImportSettings, BundleExportSettings +from nomad.config.models.config import BundleExportSettings, BundleImportSettings from nomad.files import ( - zipfile, + BrowsableFileSource, + CombinedFileSource, + DiskFileSource, + FileSource, PathObject, - UploadFiles, PublicUploadFiles, StagingUploadFiles, - FileSource, - BrowsableFileSource, - CombinedFileSource, + StandardJSONDecoder, StreamedFileSource, - DiskFileSource, + UploadFiles, ZipFileSource, - json_to_streamed_file, bundle_info_filename, - StandardJSONDecoder, + json_to_streamed_file, + zipfile, ) from nomad.processing.base import ProcessStatus -from nomad.processing.data import Upload, Entry, mongo_entry_metadata -from fastapi import HTTPException, status +from nomad.processing.data import Entry, Upload, mongo_entry_metadata class BundleExporter: diff --git a/nomad/cli/admin/admin.py b/nomad/cli/admin/admin.py index 7c02a16cdd51457e9b241def679e6d26b8a492ba..75f5cc88179cf5d44b972ddc2b668d05fa898419 100644 --- a/nomad/cli/admin/admin.py +++ b/nomad/cli/admin/admin.py @@ -18,8 +18,8 @@ import click -from nomad.config import config from nomad.cli.cli import cli +from nomad.config import config @cli.group( @@ -68,7 +68,8 @@ def reset(remove, i_am_really_sure): def reset_processing(zero_complete_time): from datetime import datetime - from nomad import infrastructure, processing as proc + from nomad import infrastructure + from nomad import processing as proc infrastructure.setup_mongo() @@ -109,9 +110,11 @@ def reset_processing(zero_complete_time): ) def lift_embargo(dry, parallel): from datetime import datetime + from dateutil.relativedelta import relativedelta - from nomad import infrastructure, processing as proc + from nomad import infrastructure + from nomad import processing as proc from nomad.search import quantity_values infrastructure.setup_mongo() @@ -415,6 +418,7 @@ def migrate_mongo( import sys from pymongo.database import Database + from nomad import infrastructure from nomad.cli.admin import migrate @@ -490,9 +494,10 @@ def migrate_mongo( ) def rewrite_doi_urls(dois, dry, save_existing_records): import json + import requests - from nomad.doi import edit_doi_url, _create_dataset_url + from nomad.doi import _create_dataset_url, edit_doi_url existing_records = [] diff --git a/nomad/cli/admin/clean.py b/nomad/cli/admin/clean.py index 6ca3d6c7ca5773b59bf79b7a716508f890d9be98..2522466c06d18e88161157447efe52e7d6264092 100644 --- a/nomad/cli/admin/clean.py +++ b/nomad/cli/admin/clean.py @@ -39,13 +39,13 @@ from .admin import admin def clean(dry, skip_entries, skip_fs, skip_es, staging_too, force): import os import shutil - import tabulate + import elasticsearch_dsl + import tabulate from nomad import infrastructure, processing from nomad.config import config as nomad_config - from nomad.search import delete_by_query - from nomad.search import quantity_values + from nomad.search import delete_by_query, quantity_values mongo_client = infrastructure.setup_mongo() infrastructure.setup_elastic() diff --git a/nomad/cli/admin/entries.py b/nomad/cli/admin/entries.py index 54ac693eed7dcd6510287497f0733e912432729a..91728ae86d3c413947cacffb9e98d7d3782efe7c 100644 --- a/nomad/cli/admin/entries.py +++ b/nomad/cli/admin/entries.py @@ -33,7 +33,8 @@ def entries(): ) @click.option('--skip-mongo', help='Keep uploads and entries in mongo.', is_flag=True) def rm(entries, skip_es, skip_mongo): - from nomad import processing as proc, infrastructure, search + from nomad import infrastructure, search + from nomad import processing as proc infrastructure.setup_mongo() infrastructure.setup_elastic() diff --git a/nomad/cli/admin/migrate.py b/nomad/cli/admin/migrate.py index 495423de7c8cc9c65c10b674bbd793e65f834aa8..5f28df89b19ae28ec8b7378d3a8545489e355b8c 100644 --- a/nomad/cli/admin/migrate.py +++ b/nomad/cli/admin/migrate.py @@ -19,15 +19,15 @@ import time from datetime import datetime from typing import Any -from pydantic import BaseModel +from pydantic import BaseModel from pymongo import ReplaceOne -from pymongo.database import Database, Collection +from pymongo.database import Collection, Database + from nomad import utils -from nomad.processing import ProcessStatus, Upload, Entry from nomad.datamodel import Dataset from nomad.parsing.parsers import parser_dict - +from nomad.processing import Entry, ProcessStatus, Upload _upload_keys_to_remove_v0 = ( 'published', diff --git a/nomad/cli/admin/run.py b/nomad/cli/admin/run.py index ab846f3e0e203bbe039094be8d0393eac403915d..1739b89c1e2dc22d887f9172f5dec9c5e68002ed 100644 --- a/nomad/cli/admin/run.py +++ b/nomad/cli/admin/run.py @@ -75,9 +75,9 @@ def run_app( # port = port or config.services.api_port if with_gui: + import glob import os import os.path - import glob import shutil gui_folder = os.path.abspath( @@ -122,9 +122,10 @@ def run_app( from nomad.utils import get_logger if gunicorn: - from gunicorn.app.wsgiapp import WSGIApplication import logging.config + from gunicorn.app.wsgiapp import WSGIApplication + if log_config: logging.config.fileConfig(log_config) @@ -152,7 +153,7 @@ def run_app( get_logger(__name__).info('created gunicorn server', data=str(gunicorn_app.cfg)) gunicorn_app.run() else: - from uvicorn import Server, Config + from uvicorn import Config, Server kwargs['log_config'] = log_config @@ -182,10 +183,11 @@ def run_worker(*, workers=None): def run_hub(): - from jupyterhub.app import main - import sys import os import subprocess + import sys + + from jupyterhub.app import main if 'JUPYTERHUB_CRYPT_KEY' not in os.environ: crypt_key = config.north.jupyterhub_crypt_key diff --git a/nomad/cli/admin/springer.py b/nomad/cli/admin/springer.py index 23b3c59e336be4b5f1745d4cb052309b875c0eb9..dec3b88fdc0c4f71670124933d6757971dceebcc 100644 --- a/nomad/cli/admin/springer.py +++ b/nomad/cli/admin/springer.py @@ -23,17 +23,18 @@ http://materials.springer.com. The database is stuctured as space_group_number : normalized_formula : springer_id : entry """ -from typing import Any -import requests +import os.path import re -import bs4 import time -import os.path +from typing import Any + +import bs4 +import requests import nomad.archive.storage_v2 from nomad import archive -from nomad.config import config from nomad.archive import read_archive +from nomad.config import config required_items = { 'Alphabetic Formula:': 'alphabetic_formula', diff --git a/nomad/cli/admin/uploads.py b/nomad/cli/admin/uploads.py index cd1cd9eb7a31698a0d147005de771d94b2502fd7..c55c64948f43df3568160bffa7438f1f9bc04783 100644 --- a/nomad/cli/admin/uploads.py +++ b/nomad/cli/admin/uploads.py @@ -25,6 +25,7 @@ import click from orjson import dumps from nomad.config import config + from .admin import admin @@ -34,7 +35,8 @@ def _run_parallel( import threading import time - from nomad import utils, processing as proc + from nomad import processing as proc + from nomad import utils if isinstance(uploads, tuple | list): uploads_count = len(uploads) @@ -237,11 +239,13 @@ def _query_uploads( list of upoad ids and further filter parameters. """ - from typing import cast import json + from typing import cast + from mongoengine import Q - from nomad import infrastructure, processing as proc, search + from nomad import infrastructure, search + from nomad import processing as proc from nomad.app.v1 import models infrastructure.setup_mongo() @@ -351,13 +355,14 @@ def _query_uploads( @click.pass_context def export(ctx, uploads, required, output: str): import sys - from nomad.processing import Entry - from nomad.utils import get_logger - from nomad.files import UploadFiles - from nomad.archive import ArchiveQueryError, RequiredReader import time import zipfile + from nomad.archive import ArchiveQueryError, RequiredReader + from nomad.files import UploadFiles + from nomad.processing import Entry + from nomad.utils import get_logger + logger = get_logger(__name__) if not output: @@ -629,7 +634,8 @@ def index(ctx, uploads, parallel, transformer, skip_materials, print_progress): def delete_upload( upload, skip_es: bool = False, skip_files: bool = False, skip_mongo: bool = False ): - from nomad import search, files, utils, processing as proc + from nomad import files, search, utils + from nomad import processing as proc # delete elastic if not skip_es: @@ -782,7 +788,8 @@ def re_pack(ctx, uploads): def stop(ctx, uploads, entries: bool, kill: bool, no_celery: bool): import mongoengine - from nomad import utils, processing as proc + from nomad import processing as proc + from nomad import utils query, _ = _query_uploads(uploads, **ctx.obj.uploads_kwargs) @@ -911,9 +918,8 @@ def integrity( ): from nomad.app.v1.models import MetadataPagination, MetadataRequired from nomad.archive.storage_v2 import ArchiveWriter - from nomad.files import StagingUploadFiles, PublicUploadFiles - from nomad.processing import Entry - from nomad.processing import Upload + from nomad.files import PublicUploadFiles, StagingUploadFiles + from nomad.processing import Entry, Upload from nomad.search import search def search_params(upload_id: str): @@ -1300,8 +1306,8 @@ def export_bundle( def import_bundle( ctx, input_path, multi, settings, embargo_length, use_celery, ignore_errors ): - from nomad.bundles import BundleImporter from nomad import infrastructure + from nomad.bundles import BundleImporter for key, value in ctx.obj.uploads_kwargs.items(): if value: diff --git a/nomad/cli/admin/users.py b/nomad/cli/admin/users.py index 21253fb313395efbbc99604d35c864f22b40e3ed..7752903f2977a4f8dc0431260abfd215c72aa258 100644 --- a/nomad/cli/admin/users.py +++ b/nomad/cli/admin/users.py @@ -29,10 +29,10 @@ def users(): @users.command(help='Import users to keycloak from a JSON file.', name='import') @click.argument('PATH_TO_USERS_FILE', type=str, nargs=1) def import_command(path_to_users_file): - import json import datetime + import json - from nomad import infrastructure, datamodel, utils + from nomad import datamodel, infrastructure, utils with open(path_to_users_file) as f: users = json.load(f) diff --git a/nomad/cli/aflow.py b/nomad/cli/aflow.py index 501b5f3764e2323bdc22bccefdf686cc1750ba4a..eec1729ad7319cf9e3cce68591a3270b9ee55de3 100644 --- a/nomad/cli/aflow.py +++ b/nomad/cli/aflow.py @@ -20,26 +20,27 @@ # code will fail. # TODO The metadata should not be set via API, but added to the uploads as nomad.json. -import requests +import io +import json +import os import re import subprocess -from urllib import parse as urllib_parse -import os import tarfile import threading import time -import io -import re import uuid -import json -import numpy as np +from urllib import parse as urllib_parse + import ase import bs4 import matid # pylint: disable=import-error +import numpy as np +import requests -from nomad import atomutils, client, processing as proc -from nomad.config import config +from nomad import atomutils, client +from nomad import processing as proc from nomad.client import api, upload_file +from nomad.config import config class DbUpdater: diff --git a/nomad/cli/cli.py b/nomad/cli/cli.py index 4120b883cc02d3b73789554071ff3c2fa4af150d..316293d1cbfc0fe6fd86009c60130b078289a95a 100644 --- a/nomad/cli/cli.py +++ b/nomad/cli/cli.py @@ -16,10 +16,11 @@ # limitations under the License. # -import click import logging import os +import click + from nomad import utils from nomad.config import config diff --git a/nomad/cli/client/integrationtests.py b/nomad/cli/client/integrationtests.py index 744cba59c894a99e755cf379fc887877e2ba088d..0f99ca2327b36d8faf2d049f1de3a4e23eed4393 100644 --- a/nomad/cli/client/integrationtests.py +++ b/nomad/cli/client/integrationtests.py @@ -21,9 +21,9 @@ A command that runs some example operations on a working nomad@FAIRDI installati as a final integration test. """ -import time -import os import json +import os +import time from nomad.client import api diff --git a/nomad/cli/dev.py b/nomad/cli/dev.py index 7a3b4a11adf3b01dc093296bd3c36bb08c426977..4491c228e9c25be42b5474982c0e95a14e4f861c 100644 --- a/nomad/cli/dev.py +++ b/nomad/cli/dev.py @@ -15,17 +15,18 @@ # limitations under the License. # -from typing import Any -import sys import json import os -import click +import sys +from typing import Any +import click from pint import Unit from pint.errors import UndefinedUnitError from nomad.config import config from nomad.metainfo.elasticsearch_extension import schema_separator + from .cli import cli @@ -80,8 +81,8 @@ def gui_qa(skip_tests: bool): @dev.command(help='Export an API model in JSON schema.') @click.argument('model') def api_model(model): - import json import importlib + import json def remove_null_types(data): """ @@ -123,11 +124,11 @@ def api_model(model): 'nomad.app.v1.models.graph.GraphRequest', 'nomad.app.v1.models.graph.GraphResponse', ]: + from nomad.app.v1.models.graph.graph_models import Graph from nomad.app.v1.models.graph.utils import ( generate_request_model, generate_response_model, ) - from nomad.app.v1.models.graph.graph_models import Graph sys.modules['nomad.app.v1.models.graph.utils'].ref_prefix = '#/definitions' sys.modules['nomad.app.v1.models.graph.utils'].graph_model_export = True @@ -188,8 +189,8 @@ def metainfo(): def _generate_search_quantities(): # Currently only quantities with "entry_type" are included. - from nomad.metainfo.elasticsearch_extension import entry_type, Elasticsearch from nomad.datamodel import EntryArchive + from nomad.metainfo.elasticsearch_extension import Elasticsearch, entry_type def to_dict(search_quantity, section=False, repeats=False): if section: @@ -349,8 +350,9 @@ def gui_config(): '--parser', help='Only updated the README of the given parsers subdirctory.' ) def update_parser_readmes(parser): - from glob import glob import re + from glob import glob + import yaml os.chdir(os.path.join(os.path.dirname(__file__), '../..')) @@ -489,8 +491,10 @@ def example_data(username: str): def _generate_units_json() -> tuple[Any, Any]: - from pint.converters import ScaleConverter from collections import defaultdict + + from pint.converters import ScaleConverter + from nomad.units import ureg # TODO: Check that all units are unambiguously defined, and that there are diff --git a/nomad/cli/parse.py b/nomad/cli/parse.py index 723796dfbac151b3372d61ca5592c47c59dcde89..275cdf80ce8c1dbd42f50c8589b2545148d48212 100644 --- a/nomad/cli/parse.py +++ b/nomad/cli/parse.py @@ -84,12 +84,12 @@ def _parse( save_plot_dir, ): import json - import sys import os + import sys + from nomad import utils from nomad.client import normalize_all, parse from nomad.datamodel.metainfo.plot import resolve_plot_references - from nomad import utils kwargs = dict( strict=not not_strict, diff --git a/nomad/client/api.py b/nomad/client/api.py index 414eb02a03ad0eb6c0cdf0aed8843cf3519d9c9c..fe85171217a799fc0f6a4d52e5959e0262580a26 100644 --- a/nomad/client/api.py +++ b/nomad/client/api.py @@ -16,9 +16,10 @@ # limitations under the License. # +import time + import requests from keycloak import KeycloakOpenID -import time from nomad.config import config diff --git a/nomad/client/archive.py b/nomad/client/archive.py index 971d87ef06408ce26484bec268894f08a93ec53c..1c0f496e09df39b92e5ba41ec22620eda40df145 100644 --- a/nomad/client/archive.py +++ b/nomad/client/archive.py @@ -18,19 +18,19 @@ from __future__ import annotations import asyncio +import threading from asyncio import Semaphore from itertools import islice -from typing import Any from time import monotonic -import threading +from typing import Any from click import progressbar -from httpx import Timeout, AsyncClient +from httpx import AsyncClient, Timeout from keycloak import KeycloakOpenID from nomad import metainfo as mi from nomad.config import config -from nomad.datamodel import EntryArchive, ClientContext +from nomad.datamodel import ClientContext, EntryArchive from nomad.utils import dict_to_dataframe diff --git a/nomad/client/processing.py b/nomad/client/processing.py index 3445fc88711a48284f432cd15507ac0c93955adb..03d54ba78cd8bd8ec47892cd42d83a7ea02c711c 100644 --- a/nomad/client/processing.py +++ b/nomad/client/processing.py @@ -16,15 +16,14 @@ # limitations under the License. # -import os import io -import typing +import os import sys +import typing -from nomad import utils, datamodel +from nomad import datamodel, utils from nomad.config import config - from .api import Auth diff --git a/nomad/client/upload.py b/nomad/client/upload.py index e800b8da04781ad004150d26b8cc47042d5917e6..8b2c6f4c38d1e974cd1b72637338df1c1854cf6f 100644 --- a/nomad/client/upload.py +++ b/nomad/client/upload.py @@ -41,8 +41,8 @@ def upload_file( Returns: The upload_id if successful or None if not. """ - from nomad.processing import ProcessStatus from nomad.client import api + from nomad.processing import ProcessStatus if local_path: response = api.post( diff --git a/nomad/common.py b/nomad/common.py index e3f7bc67a3ea072f581100f0bf74d6e94c6a7185..181c232cbe600ea4f0914ace29c5a710f7cb4825 100644 --- a/nomad/common.py +++ b/nomad/common.py @@ -24,10 +24,11 @@ source code without circular imports. import os import pkgutil import shutil -import zipfile import tarfile -from typing import Literal +import zipfile from tempfile import TemporaryDirectory +from typing import Literal + import httpx diff --git a/nomad/config/models/common.py b/nomad/config/models/common.py index 576c4aa03d0852346485a6922f877f5f86d98900..02f68c3814f991baaaf0f26089e97ef342817919 100644 --- a/nomad/config/models/common.py +++ b/nomad/config/models/common.py @@ -16,8 +16,9 @@ # limitations under the License. # -from typing import Any, cast, TypeVar -from pydantic import ConfigDict, model_validator, BaseModel, Field # noqa: F401 +from typing import Any, TypeVar, cast + +from pydantic import BaseModel, ConfigDict, Field, model_validator # noqa: F401 ConfigBaseModelBound = TypeVar('ConfigBaseModelBound', bound='ConfigBaseModel') diff --git a/nomad/config/models/config.py b/nomad/config/models/config.py index ffba90ddc9af27b13d51d36c04854eb913e716ae..385f0d8ba62b059589d96448b1e237d1b804b0c7 100644 --- a/nomad/config/models/config.py +++ b/nomad/config/models/config.py @@ -23,13 +23,7 @@ from importlib.metadata import version from typing import Any import yaml -from pydantic import ( - BaseModel, - field_validator, - model_validator, - Field, - ConfigDict, -) +from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator try: __version__ = version('nomad-lab') @@ -39,15 +33,12 @@ except Exception: # noqa from importlib.metadata import entry_points +from nomad.common import get_package_path -from .common import ( - ConfigBaseModel, - Options, -) +from .common import ConfigBaseModel, Options from .north import NORTH from .plugins import EntryPointType, PluginPackage, Plugins from .ui import UI -from nomad.common import get_package_path warnings.filterwarnings('ignore', message='numpy.dtype size changed') warnings.filterwarnings('ignore', message='numpy.ufunc size changed') diff --git a/nomad/config/models/plugins.py b/nomad/config/models/plugins.py index d0fe2af9f0073fe64639af4b343f2252c8c563b1..6ab41803294bc867da0327930d7c622ef6859fa4 100644 --- a/nomad/config/models/plugins.py +++ b/nomad/config/models/plugins.py @@ -16,15 +16,16 @@ # limitations under the License. # +import importlib import os -import sys import shutil +import sys from abc import ABCMeta, abstractmethod -import importlib -from typing import Union, Literal, cast, TYPE_CHECKING -from pydantic import model_validator, BaseModel, Field +from typing import TYPE_CHECKING, Literal, Union, cast + +from pydantic import BaseModel, Field, model_validator -from nomad.common import get_package_path, download_file, is_url, is_safe_relative_path +from nomad.common import download_file, get_package_path, is_safe_relative_path, is_url from .common import Options from .ui import App @@ -32,10 +33,11 @@ from .ui import App example_prefix = '__examples__' if TYPE_CHECKING: + from fastapi import FastAPI + from nomad.metainfo import SchemaPackage from nomad.normalizing import Normalizer as NormalizerBaseClass from nomad.parsing import Parser as ParserBaseClass - from fastapi import FastAPI class EntryPoint(BaseModel): @@ -762,8 +764,8 @@ def add_plugin(plugin: Schema) -> None: def remove_plugin(plugin) -> None: """Function for removing a plugin.""" from nomad.config import config - from nomad.metainfo.elasticsearch_extension import entry_type from nomad.metainfo import Package + from nomad.metainfo.elasticsearch_extension import entry_type # Remove from path try: diff --git a/nomad/config/models/ui.py b/nomad/config/models/ui.py index 3e42990c93a731be07e070eb451349c01641405e..ed171ebd81925128e2e02dcf960a5d2483f900f3 100644 --- a/nomad/config/models/ui.py +++ b/nomad/config/models/ui.py @@ -17,18 +17,17 @@ # from enum import Enum -from typing import Union -from typing import Literal -from typing import Annotated -from pydantic import BaseModel, ConfigDict, model_validator, Field +from typing import Annotated, Literal, Union + +from pydantic import BaseModel, ConfigDict, Field, model_validator from .common import ( ConfigBaseModel, Options, - OptionsSingle, - OptionsMulti, - OptionsGlob, OptionsBase, + OptionsGlob, + OptionsMulti, + OptionsSingle, ) @@ -116,9 +115,10 @@ class UnitSystem(ConfigBaseModel): values = values.model_dump(exclude_none=True) """Adds SI defaults for dimensions that are missing a unit.""" units = values.get('units', {}) - from nomad.units import ureg from pint import UndefinedUnitError + from nomad.units import ureg + # Check that only supported dimensions and units are used for key in units.keys(): if key not in dimensions: diff --git a/nomad/datamodel/context.py b/nomad/datamodel/context.py index e9ca61a9ee0868cc046367e9553b37a71d420e1e..3c4529c903c1b6107159863d0976416c76390676 100644 --- a/nomad/datamodel/context.py +++ b/nomad/datamodel/context.py @@ -16,24 +16,19 @@ # limitations under the License. # -from urllib.parse import urlsplit, urlunsplit -import re import os.path +import re +from urllib.parse import urlsplit, urlunsplit import requests from nomad import utils from nomad.config import config -from nomad.datamodel.util import parse_path -from nomad.datamodel.datamodel import EntryMetadata -from nomad.metainfo import ( - Context as MetainfoContext, - MSection, - Quantity, - MetainfoReferenceError, - Package, -) from nomad.datamodel import EntryArchive +from nomad.datamodel.datamodel import EntryMetadata +from nomad.datamodel.util import parse_path +from nomad.metainfo import Context as MetainfoContext +from nomad.metainfo import MetainfoReferenceError, MSection, Package, Quantity class Context(MetainfoContext): diff --git a/nomad/datamodel/data.py b/nomad/datamodel/data.py index 595156644c733089da5dc12f9359004bb63f9dd0..ca029f896327fc8748978d8c9380f74d5ab4a41e 100644 --- a/nomad/datamodel/data.py +++ b/nomad/datamodel/data.py @@ -17,24 +17,23 @@ # import os.path +from typing import Any from cachetools import TTLCache, cached - -from typing import Any from pydantic import Field from nomad.config import config from nomad.metainfo.elasticsearch_extension import Elasticsearch, material_entry_type from nomad.metainfo.metainfo import ( + JSON, + Capitalized, Category, + Datetime, MCategory, MSection, Quantity, - Capitalized, - Section, - Datetime, Reference, - JSON, + Section, ) from nomad.metainfo.pydantic_extension import PydanticModel @@ -96,8 +95,8 @@ class EntryData(ArchiveSection): def normalize(self, archive, logger): super().normalize(archive, logger) - from nomad.datamodel.results import Results from nomad.datamodel import EntryArchive + from nomad.datamodel.results import Results # TODO entry_type should only be assigned if not already defined (done to pass eln test) if archive.metadata: diff --git a/nomad/datamodel/datamodel.py b/nomad/datamodel/datamodel.py index f32012bc14297a83dec7bcbcf9338c5caf4b1789..3b048eec5182b6980d3b40a9dabe92e5194a4d80 100644 --- a/nomad/datamodel/datamodel.py +++ b/nomad/datamodel/datamodel.py @@ -18,39 +18,40 @@ """All generic entry metadata and related classes.""" -from typing import Any -from enum import Enum import os.path +from enum import Enum +from typing import Any import rfc3161ng from elasticsearch_dsl import analyzer, tokenizer from nomad import utils from nomad.datamodel.metainfo.common import FastAccess -from nomad.metainfo.mongoengine_extension import Mongo, MongoDocument -from nomad.metainfo.pydantic_extension import PydanticModel from nomad.metainfo.elasticsearch_extension import ( Elasticsearch, - material_entry_type, - entry_type as es_entry_type, create_searchable_quantity, + material_entry_type, ) -from .util import parse_path +from nomad.metainfo.elasticsearch_extension import entry_type as es_entry_type +from nomad.metainfo.mongoengine_extension import Mongo, MongoDocument +from nomad.metainfo.pydantic_extension import PydanticModel + from ..metainfo import ( + JSON, Bytes, - Package, + Datetime, Definition, - MSection, MCategory, + MEnum, + MSection, + Package, + Quantity, Section, SubSection, - Quantity, - MEnum, - Datetime, - JSON, ) from ..metainfo.data_type import m_str from ..metainfo.metainfo import Reference +from .util import parse_path # This is usually defined automatically when the first metainfo definition is evaluated, but # due to the next imports requiring the m_package already, this would be too late. diff --git a/nomad/datamodel/hdf5.py b/nomad/datamodel/hdf5.py index fa04a76f9465b26cfa6eb6f0271cd40e7f0cd396..daf56c68ad8fdd8de9750f7de76a07d8abff82f5 100644 --- a/nomad/datamodel/hdf5.py +++ b/nomad/datamodel/hdf5.py @@ -17,16 +17,16 @@ # from __future__ import annotations -from typing import Any -import h5py import re +from typing import Any +import h5py import numpy as np import pint from h5py import File -from nomad.metainfo.data_type import NonPrimitive from nomad.datamodel.metainfo.annotations import H5WebAnnotation +from nomad.metainfo.data_type import NonPrimitive from nomad.utils import get_logger LOGGER = get_logger(__name__) diff --git a/nomad/datamodel/metainfo/action.py b/nomad/datamodel/metainfo/action.py index a4c73d3de524f16e174415d663477c1bbde37261..8f219878de54194a6c1eb9040efbd3a7a4f8e184 100644 --- a/nomad/datamodel/metainfo/action.py +++ b/nomad/datamodel/metainfo/action.py @@ -16,7 +16,7 @@ # limitations under the License. # from nomad.datamodel.data import ArchiveSection -from nomad.metainfo import Quantity, Package +from nomad.metainfo import Package, Quantity m_package = Package() diff --git a/nomad/datamodel/metainfo/annotations.py b/nomad/datamodel/metainfo/annotations.py index fb9fc113241354a6f1c61d62a15a4ce9f795d048..ed177b2541d635769a755fa10fc544f220765c29 100644 --- a/nomad/datamodel/metainfo/annotations.py +++ b/nomad/datamodel/metainfo/annotations.py @@ -16,18 +16,19 @@ # limitations under the License. # -from typing import Any -from enum import Enum -from pydantic import field_validator, ConfigDict, Field, model_validator import re +from enum import Enum +from typing import Any +from pydantic import ConfigDict, Field, field_validator, model_validator from pydantic.main import BaseModel +from nomad.metainfo import AnnotationModel, Datetime, MEnum, Quantity, Reference from nomad.utils import strip -from nomad.metainfo import AnnotationModel, MEnum, Datetime, Reference, Quantity -from .plot import PlotlyError -from ..data import Query + from ...metainfo.data_type import Datatype +from ..data import Query +from .plot import PlotlyError class ELNComponentEnum(str, Enum): diff --git a/nomad/datamodel/metainfo/basesections/v1.py b/nomad/datamodel/metainfo/basesections/v1.py index 69cebd02de08bdcd4b554099203208030662b3f0..eacde4ab5eab8fb1ba55d743ad57606d1ef5b009 100644 --- a/nomad/datamodel/metainfo/basesections/v1.py +++ b/nomad/datamodel/metainfo/basesections/v1.py @@ -20,8 +20,8 @@ import os import random import re import time -from typing import TYPE_CHECKING from collections.abc import Iterable +from typing import TYPE_CHECKING import h5py import numpy as np @@ -29,22 +29,17 @@ import requests from ase.data import atomic_masses, atomic_numbers, chemical_symbols from unidecode import unidecode -from nomad.metainfo import SchemaPackage from nomad.datamodel.metainfo.workflow import Link, Task, TaskReference, Workflow +from nomad.metainfo import SchemaPackage from nomad.metainfo.data_type import m_str if TYPE_CHECKING: - from structlog.stdlib import ( - BoundLogger, - ) + from structlog.stdlib import BoundLogger from nomad import utils from nomad.atomutils import Formula from nomad.datamodel.data import ArchiveSection, EntryData -from nomad.datamodel.metainfo.annotations import ( - ELNAnnotation, - HDF5Annotation, -) +from nomad.datamodel.metainfo.annotations import ELNAnnotation, HDF5Annotation from nomad.datamodel.results import ELN, Material, Results from nomad.datamodel.results import ElementalComposition as ResultsElementalComposition from nomad.datamodel.util import create_custom_mapping @@ -2039,6 +2034,7 @@ class PublicationReference(ArchiveSection): super().normalize(archive, logger) import dateutil.parser import requests + from nomad.datamodel.datamodel import EntryMetadata # Parse journal name, lead author and publication date from crossref diff --git a/nomad/datamodel/metainfo/basesections/v2.py b/nomad/datamodel/metainfo/basesections/v2.py index 01aad791e139f31922c58306c6d2cb353daa03a3..1ad7e401f79d68d49a8ba972424a624f9e427b15 100644 --- a/nomad/datamodel/metainfo/basesections/v2.py +++ b/nomad/datamodel/metainfo/basesections/v2.py @@ -15,62 +15,39 @@ # See the License for the specific language governing permissions and # limitations under the License. # +import datetime import os -from typing import TYPE_CHECKING -from collections.abc import Iterable import random -import time -import datetime import re +import time +from collections.abc import Iterable +from typing import TYPE_CHECKING -from unidecode import unidecode -import numpy as np import h5py -from ase.data import ( - chemical_symbols, - atomic_numbers, - atomic_masses, -) +import numpy as np import requests +from ase.data import atomic_masses, atomic_numbers, chemical_symbols +from unidecode import unidecode from nomad.datamodel.metainfo.workflow import Link, Task, Workflow from nomad.metainfo.data_type import m_str if TYPE_CHECKING: - from structlog.stdlib import ( - BoundLogger, - ) -from nomad import ( - utils, -) -from nomad.units import ( - ureg, -) -from nomad.metainfo import ( - Quantity, - Datetime, - Section, - SectionProxy, - SubSection, -) -from nomad.metainfo.util import MEnum -from nomad.datamodel.util import create_custom_mapping -from nomad.datamodel.data import ( - ArchiveSection, -) -from nomad.datamodel.results import ( - Results, - ELN, - ElementalComposition as ResultsElementalComposition, - Material, -) + from structlog.stdlib import BoundLogger +from nomad import utils +from nomad.datamodel.data import ArchiveSection from nomad.datamodel.metainfo.annotations import ( ELNAnnotation, Filter, - SectionProperties, HDF5Annotation, + SectionProperties, ) - +from nomad.datamodel.results import ELN, Material, Results +from nomad.datamodel.results import ElementalComposition as ResultsElementalComposition +from nomad.datamodel.util import create_custom_mapping +from nomad.metainfo import Datetime, Quantity, Section, SectionProxy, SubSection +from nomad.metainfo.util import MEnum +from nomad.units import ureg PUB_CHEM_PUG_PATH = 'https://pubchem.ncbi.nlm.nih.gov/rest/pug/compound' CAS_API_PATH = 'https://commonchemistry.cas.org/api' @@ -408,7 +385,7 @@ class EntityReference(SectionReference): """ super().normalize(archive, logger) if self.reference is None and self.lab_id is not None: - from nomad.search import search, MetadataPagination + from nomad.search import MetadataPagination, search query = {'results.eln.lab_ids': self.lab_id} search_result = search( @@ -1595,10 +1572,11 @@ class PublicationReference(ArchiveSection): logger ('BoundLogger'): A structlog logger. """ super().normalize(archive, logger) - from nomad.datamodel.datamodel import EntryMetadata import dateutil.parser import requests + from nomad.datamodel.datamodel import EntryMetadata + # Parse journal name, lead author and publication date from crossref if self.DOI_number: try: diff --git a/nomad/datamodel/metainfo/common.py b/nomad/datamodel/metainfo/common.py index 14bc1024ec90ec024b8bb863729254b268c6dbfa..beeb755d6d910afe3250dbe525b763c6ed54a4b2 100644 --- a/nomad/datamodel/metainfo/common.py +++ b/nomad/datamodel/metainfo/common.py @@ -16,8 +16,7 @@ # limitations under the License. # -from nomad.metainfo import MCategory, Category -from nomad.metainfo import MSection, Section, SubSection, Quantity +from nomad.metainfo import Category, MCategory, MSection, Quantity, Section, SubSection from nomad.metainfo.elasticsearch_extension import Elasticsearch, material_entry_type diff --git a/nomad/datamodel/metainfo/downloads.py b/nomad/datamodel/metainfo/downloads.py index 9ff64389d79966909ddf158a38a2cc249d61f781..8ada69e13bc3735549b3955517b4a5ccd762af1e 100644 --- a/nomad/datamodel/metainfo/downloads.py +++ b/nomad/datamodel/metainfo/downloads.py @@ -18,10 +18,8 @@ import os.path -from nomad.metainfo import MSection, Package, Quantity, SubSection - from nomad.datamodel.data import ArchiveSection - +from nomad.metainfo import MSection, Package, Quantity, SubSection m_package = Package(name='downloads') @@ -137,7 +135,8 @@ class Downloads(ArchiveSection): import pathlib import urllib.request - from nomad.common import get_compression_format, extract_file + + from nomad.common import extract_file, get_compression_format # download and extract files skip_download = True diff --git a/nomad/datamodel/metainfo/measurements.py b/nomad/datamodel/metainfo/measurements.py index c10787470caf5c12a1fde04e29fd44a6daa80698..a6bb1fca5cf666c4c9699d8b15d3ec1228248011 100644 --- a/nomad/datamodel/metainfo/measurements.py +++ b/nomad/datamodel/metainfo/measurements.py @@ -18,10 +18,9 @@ import numpy as np -from nomad.metainfo import MSection, Package, Quantity, SubSection, Datetime -from nomad.metainfo.metainfo import Reference, SectionProxy from nomad.datamodel import Author - +from nomad.metainfo import Datetime, MSection, Package, Quantity, SubSection +from nomad.metainfo.metainfo import Reference, SectionProxy m_package = Package(name='measurements') diff --git a/nomad/datamodel/metainfo/plot.py b/nomad/datamodel/metainfo/plot.py index 3618263f8e78b5f19f434f6af17791de836de6a4..caaf30037f341d1d9038009d2d75094d12a85b2a 100644 --- a/nomad/datamodel/metainfo/plot.py +++ b/nomad/datamodel/metainfo/plot.py @@ -15,14 +15,16 @@ # See the License for the specific language governing permissions and # limitations under the License. # -from nomad.datamodel.data import ArchiveSection -from nomad.metainfo import Quantity, SubSection, Package, MSection, JSON, Section +from copy import deepcopy +from datetime import datetime + +import numpy as np import plotly.express as px import plotly.graph_objs as go from plotly.subplots import make_subplots -import numpy as np -from copy import deepcopy -from datetime import datetime + +from nomad.datamodel.data import ArchiveSection +from nomad.metainfo import JSON, MSection, Package, Quantity, Section, SubSection class PlotlyError(Exception): diff --git a/nomad/datamodel/metainfo/simulation/calculation.py b/nomad/datamodel/metainfo/simulation/calculation.py index 1dc907a8eefc03bc6bb1a42692179f29e2234103..a4ecc9095459db3682c480b3cfcd6b9693620f14 100644 --- a/nomad/datamodel/metainfo/simulation/calculation.py +++ b/nomad/datamodel/metainfo/simulation/calculation.py @@ -21,27 +21,27 @@ # https://github.com/nomad-coe/nomad-schema-plugin-run.git import numpy as np # noqa: F401 + +from nomad.datamodel.data import ArchiveSection +from nomad.datamodel.metainfo.common import PropertySection, ProvenanceTracker +from nomad.datamodel.metainfo.simulation.method import HoppingMatrix, Method +from nomad.datamodel.metainfo.simulation.system import AtomsGroup, System from nomad.metainfo import ( # noqa: F401 - MSection, - MCategory, Category, + MCategory, + MEnum, + MSection, Package, Quantity, + Reference, Section, - SubSection, SectionProxy, - Reference, - MEnum, + SubSection, derived, ) -from nomad.datamodel.metainfo.common import ProvenanceTracker, PropertySection -from nomad.datamodel.metainfo.simulation.system import System, AtomsGroup -from nomad.datamodel.metainfo.simulation.method import Method, HoppingMatrix -from nomad.datamodel.data import ArchiveSection from ..common import FastAccess - m_package = Package() diff --git a/nomad/datamodel/metainfo/simulation/legacy_workflows.py b/nomad/datamodel/metainfo/simulation/legacy_workflows.py index 52a62e4a1f4833c630a4ef2202e7a739863088ac..7a4eef3e5d887f275b911570155e4d03d2b81b4c 100644 --- a/nomad/datamodel/metainfo/simulation/legacy_workflows.py +++ b/nomad/datamodel/metainfo/simulation/legacy_workflows.py @@ -22,26 +22,26 @@ import numpy as np from nptyping import NDArray + +from nomad.datamodel.metainfo.common import FastAccess +from nomad.datamodel.metainfo.simulation.calculation import ( + BandStructure, + Calculation, + Dos, +) +from nomad.datamodel.metainfo.simulation.run import Run +from nomad.datamodel.metainfo.simulation.system import Atoms, AtomsGroup, System from nomad.metainfo import ( - MSection, MEnum, + MSection, + Package, Quantity, + Reference, Section, - SubSection, SectionProxy, - Reference, - Package, + SubSection, derived, ) -from nomad.datamodel.metainfo.simulation.calculation import ( - Calculation, - Dos, - BandStructure, -) -from nomad.datamodel.metainfo.simulation.run import Run -from nomad.datamodel.metainfo.simulation.system import System, Atoms, AtomsGroup -from nomad.datamodel.metainfo.common import FastAccess - m_package = Package() diff --git a/nomad/datamodel/metainfo/simulation/method.py b/nomad/datamodel/metainfo/simulation/method.py index 165fdfa0ae05dcab3ac0fd245e7dcd730cfda1e0..0d88668563fd6c5dcbd90c76a15f92d0e65381d9 100644 --- a/nomad/datamodel/metainfo/simulation/method.py +++ b/nomad/datamodel/metainfo/simulation/method.py @@ -20,27 +20,28 @@ # Only for purpose of compatibility. Use run schema plugin. # https://github.com/nomad-coe/nomad-schema-plugin-run.git -from logging import Logger -import numpy as np # noqa: F401 import typing +from logging import Logger +import numpy as np # noqa: F401 from pint.util import SharedRegistryObject # noqa: F401 + from nomad.datamodel.data import ArchiveSection from nomad.metainfo import ( # noqa: F401 - MSection, - MCategory, Category, + MCategory, + MEnum, + MSection, Package, Quantity, + Reference, Section, - SubSection, SectionProxy, - Reference, - MEnum, + SubSection, ) from nomad.quantum_states import RussellSaundersState -from ..common import FastAccess +from ..common import FastAccess m_package = Package() diff --git a/nomad/datamodel/metainfo/simulation/run.py b/nomad/datamodel/metainfo/simulation/run.py index 93cca85f6a29e593ccd2de76e606b459818a1c33..2dd64bf26696ef6d4b7450071f435851a39a8eed 100644 --- a/nomad/datamodel/metainfo/simulation/run.py +++ b/nomad/datamodel/metainfo/simulation/run.py @@ -22,22 +22,22 @@ import numpy as np # noqa: F401 +from nomad.datamodel.data import ArchiveSection +from nomad.datamodel.metainfo.common import FastAccess +from nomad.datamodel.metainfo.simulation.calculation import Calculation +from nomad.datamodel.metainfo.simulation.method import Method +from nomad.datamodel.metainfo.simulation.system import System from nomad.metainfo import ( # noqa: F401 - MSection, - MCategory, Category, + MCategory, + MSection, Package, Quantity, + Reference, Section, - SubSection, SectionProxy, - Reference, + SubSection, ) -from nomad.datamodel.metainfo.simulation.method import Method -from nomad.datamodel.metainfo.simulation.system import System -from nomad.datamodel.metainfo.simulation.calculation import Calculation -from nomad.datamodel.metainfo.common import FastAccess -from nomad.datamodel.data import ArchiveSection m_package = Package() diff --git a/nomad/datamodel/metainfo/simulation/system.py b/nomad/datamodel/metainfo/simulation/system.py index fe4a73deab62e75a8de3a205f123976c6f73284d..c5645aebbf0f6aa5dd19826b6213f13a52f88019 100644 --- a/nomad/datamodel/metainfo/simulation/system.py +++ b/nomad/datamodel/metainfo/simulation/system.py @@ -20,26 +20,28 @@ # Only for purpose of compatibility. Use run schema plugin. # https://github.com/nomad-coe/nomad-schema-plugin-run.git -import numpy as np # noqa: F401 import typing # noqa: F401 + +import numpy as np # noqa: F401 + +from nomad.datamodel.data import ArchiveSection from nomad.metainfo import ( # noqa: F401 - MSection, - MCategory, Category, + MCategory, + MEnum, + MSection, Package, Quantity, + Reference, Section, - SubSection, SectionProxy, - Reference, - MEnum, + SubSection, derived, ) -from nomad.datamodel.data import ArchiveSection from nomad.metainfo.data_type import m_float64 +from nomad.units import ureg from ..common import FastAccess -from nomad.units import ureg m_package = Package() diff --git a/nomad/datamodel/metainfo/simulation/workflow.py b/nomad/datamodel/metainfo/simulation/workflow.py index ae5dde4f1b2eef7c68a5c61fef655413506721f8..aa77ddb2d01bccb48645324e4d9986c897058ba3 100644 --- a/nomad/datamodel/metainfo/simulation/workflow.py +++ b/nomad/datamodel/metainfo/simulation/workflow.py @@ -27,45 +27,48 @@ from nptyping import NDArray from nomad.atomutils import get_volume from nomad.datamodel.data import ArchiveSection -from nomad.units import ureg -from nomad.metainfo import ( - MSection, - SubSection, - Section, - Quantity, - MEnum, - Reference, - Package, - derived, -) from nomad.datamodel.metainfo.common import FastAccess -from nomad.datamodel.metainfo.workflow import Workflow, Link, Task -from nomad.datamodel.metainfo.simulation.system import System, AtomsGroup -from nomad.datamodel.metainfo.simulation.method import ( - Method, - XCFunctional, - BasisSetContainer, - GW as GWMethodology, - TB as TBMethodology, - DMFT as DMFTMethodology, - BSE as BSEMethodology, -) from nomad.datamodel.metainfo.simulation.calculation import ( - Calculation, + BandEnergies, BandGap, - Dos, BandStructure, - BandEnergies, + Calculation, Density, - Potential, - Spectra, + Dos, ElectronicStructureProvenance, + EnergyEntry, GreensFunctions, + Potential, + Spectra, +) +from nomad.datamodel.metainfo.simulation.calculation import ( RadiusOfGyration as RadiusOfGyrationCalculation, +) +from nomad.datamodel.metainfo.simulation.calculation import ( RadiusOfGyrationValues as RadiusOfGyrationValuesCalculation, - EnergyEntry, ) - +from nomad.datamodel.metainfo.simulation.method import BSE as BSEMethodology +from nomad.datamodel.metainfo.simulation.method import DMFT as DMFTMethodology +from nomad.datamodel.metainfo.simulation.method import GW as GWMethodology +from nomad.datamodel.metainfo.simulation.method import TB as TBMethodology +from nomad.datamodel.metainfo.simulation.method import ( + BasisSetContainer, + Method, + XCFunctional, +) +from nomad.datamodel.metainfo.simulation.system import AtomsGroup, System +from nomad.datamodel.metainfo.workflow import Link, Task, Workflow +from nomad.metainfo import ( + MEnum, + MSection, + Package, + Quantity, + Reference, + Section, + SubSection, + derived, +) +from nomad.units import ureg # TODO remove this after reprocessing with the new schema defined in # simulationworkflowschema plug in https://github.com/nomad-coe/nomad-schema-plugin-simulation-workflow.git @@ -2202,11 +2205,11 @@ class MolecularDynamicsResults(ThermodynamicsResults): super().normalize(archive, logger) try: - from simulationworkflowschema.molecular_dynamics import archive_to_universe from simulationworkflowschema.molecular_dynamics import ( - calc_molecular_rdf, + archive_to_universe, calc_molecular_mean_squared_displacements, calc_molecular_radius_of_gyration, + calc_molecular_rdf, ) universe = archive_to_universe(archive) diff --git a/nomad/datamodel/metainfo/system.py b/nomad/datamodel/metainfo/system.py index bcff5a8172b802abbdf02991f982c954c32b47b2..05fce8c21c63a8dc2c831dde8a8647fef4c31ca0 100644 --- a/nomad/datamodel/metainfo/system.py +++ b/nomad/datamodel/metainfo/system.py @@ -15,11 +15,11 @@ # See the License for the specific language governing permissions and # limitations under the License. # -import numpy as np import ase +import numpy as np -from nomad.metainfo import Package, Quantity, Section, SubSection, SectionProxy from nomad.datamodel.data import ArchiveSection +from nomad.metainfo import Package, Quantity, Section, SectionProxy, SubSection from nomad.units import ureg # TODO System should be redefined from base section diff --git a/nomad/datamodel/metainfo/tabulartree.py b/nomad/datamodel/metainfo/tabulartree.py index 806be1c2c6651d346cf82af29a3006dd3b66984f..496090623503842216ee0543243152407caa7f84 100644 --- a/nomad/datamodel/metainfo/tabulartree.py +++ b/nomad/datamodel/metainfo/tabulartree.py @@ -20,7 +20,6 @@ import typing from nomad.metainfo import MSection, Package, Quantity, SubSection - m_package = Package(name='tabulartree') diff --git a/nomad/datamodel/metainfo/workflow.py b/nomad/datamodel/metainfo/workflow.py index 2f446ebaeec2da83ff31e47e2809bf03d0dc5971..a1c3fe137a10327b3ec04aeb1ad514d736e9c8b1 100644 --- a/nomad/datamodel/metainfo/workflow.py +++ b/nomad/datamodel/metainfo/workflow.py @@ -16,9 +16,8 @@ # limitations under the License. # -from nomad.metainfo import Quantity, SubSection, Section - from nomad.datamodel.data import ArchiveSection, EntryData, WorkflowsElnCategory +from nomad.metainfo import Quantity, Section, SubSection class Link(ArchiveSection): diff --git a/nomad/datamodel/optimade.py b/nomad/datamodel/optimade.py index f72fa69cbf3a836fc54d7b3ab0af320716d91868..5dfea496884e1083320992269c16d073e97cdb3e 100644 --- a/nomad/datamodel/optimade.py +++ b/nomad/datamodel/optimade.py @@ -16,19 +16,19 @@ # limitations under the License. # -from ase.data import chemical_symbols import numpy as np +from ase.data import chemical_symbols -from nomad.units import ureg from nomad.metainfo import ( + DefinitionAnnotation, + MEnum, MSection, - Section, Quantity, + Section, SubSection, - MEnum, - DefinitionAnnotation, ) from nomad.metainfo.elasticsearch_extension import Elasticsearch +from nomad.units import ureg def optimade_links(section: str): diff --git a/nomad/datamodel/results.py b/nomad/datamodel/results.py index f4f090d2ba4e7b46c5d5cedfc30a17257a4ba676..2942b4bd44cfe49ccaeae5cae4dfd147b145d576 100644 --- a/nomad/datamodel/results.py +++ b/nomad/datamodel/results.py @@ -18,43 +18,40 @@ from logging import Logger from typing import TYPE_CHECKING -import numpy as np -from elasticsearch_dsl import Text +import numpy as np from ase.data import chemical_symbols +from elasticsearch_dsl import Text from nomad.config import config -from nomad.datamodel.metainfo.common import ProvenanceTracker, PropertySection +from nomad.datamodel.metainfo.annotations import H5WebAnnotation +from nomad.datamodel.metainfo.common import PropertySection, ProvenanceTracker from nomad.datamodel.metainfo.simulation.method import CoreHole as CoreHoleRun -from nomad.metainfo.elasticsearch_extension import ( - Elasticsearch, - material_type, - material_entry_type, - get_tokenizer, -) - +from nomad.datamodel.optimade import Species as OptimadeSpecies # noqa from nomad.metainfo import ( - MSection, - Section, - SubSection, - Quantity, + Datetime, MEnum, + MSection, Package, - Datetime, + Quantity, Reference, + Section, + SubSection, +) +from nomad.metainfo.elasticsearch_extension import ( + Elasticsearch, + get_tokenizer, + material_entry_type, + material_type, ) -from nomad.datamodel.metainfo.common import ProvenanceTracker, PropertySection -from nomad.datamodel.optimade import Species as OptimadeSpecies # noqa -from nomad.datamodel.metainfo.annotations import H5WebAnnotation try: import runschema runschema.run_schema_entry_point.load() - import runschema.method import runschema.calculation + import runschema.method import runschema.system - import simulationworkflowschema simulationworkflowschema.simulationworkflow_schema_entry_point.load() diff --git a/nomad/datamodel/util.py b/nomad/datamodel/util.py index 63ab9e0c481a61278f3dbee0373d5beb7626780e..60780338245abd42e1bfc44f3715804742c4a8f9 100644 --- a/nomad/datamodel/util.py +++ b/nomad/datamodel/util.py @@ -17,19 +17,19 @@ # import math import re -from typing import Any from collections.abc import Callable +from typing import Any import numpy as np from nomad import utils from nomad.metainfo import ( - Section, AnnotationModel, + MetainfoError, MSection, - SubSection, Property, - MetainfoError, + Section, + SubSection, ) from nomad.units import ureg diff --git a/nomad/doi.py b/nomad/doi.py index a755e19c31e6059503dbef60b6d59340b954bbda..fe1df5f0492c2500fee31111c48572c19ab58f76 100644 --- a/nomad/doi.py +++ b/nomad/doi.py @@ -21,17 +21,18 @@ This module contains all functions necessary to manage DOI via datacite.org and MDS API (https://support.datacite.org/docs/mds-api-guide). """ -import xml.etree.ElementTree as ET import datetime +import xml.etree.ElementTree as ET + import requests -from requests.auth import HTTPBasicAuth -from mongoengine import Document, StringField, DateTimeField +from fastapi import HTTPException +from mongoengine import DateTimeField, Document, StringField from mongoengine.errors import NotUniqueError +from requests.auth import HTTPBasicAuth -from nomad.datamodel import User -from nomad.config import config from nomad import utils -from fastapi import HTTPException +from nomad.config import config +from nomad.datamodel import User class DOIException(Exception): diff --git a/nomad/files.py b/nomad/files.py index fdf0f0b8ee144a08c552cf4717b79ed77e58bfab..d40bf0e1024af97eaaf1339d03d6c8021c39d542 100644 --- a/nomad/files.py +++ b/nomad/files.py @@ -45,40 +45,35 @@ original mainfile, and vice versa. from __future__ import annotations -from abc import ABCMeta -from typing import ( - IO, - Any, - NamedTuple, -) -from collections.abc import Callable -from collections.abc import Iterable, Iterator -from pydantic import BaseModel -from datetime import datetime -import os.path -import os -import shutil -import zipstream import hashlib import io import json -import yaml -import magic +import os +import os.path +import shutil import tarfile # noqa: F401 import zipfile +from abc import ABCMeta +from collections.abc import Callable, Iterable, Iterator +from datetime import datetime +from typing import IO, Any, NamedTuple +import magic +import yaml +import zipstream +from pydantic import BaseModel -from nomad import utils, datamodel +from nomad import datamodel, utils +from nomad.archive import ArchiveReader, read_archive, to_json, write_archive +from nomad.archive.storage_v2 import combine_archive from nomad.common import ( - get_compression_format, extract_file, + get_compression_format, is_safe_basename, is_safe_relative_path, ) from nomad.config import config -from nomad.archive.storage_v2 import combine_archive -from nomad.config.models.config import BundleImportSettings, BundleExportSettings -from nomad.archive import write_archive, read_archive, ArchiveReader, to_json +from nomad.config.models.config import BundleExportSettings, BundleImportSettings bundle_info_filename = 'bundle_info.json' diff --git a/nomad/graph/graph_reader.py b/nomad/graph/graph_reader.py index d813a8f3755159a98aa44e1533793f4f0645ea43..24139f771409b93ca2c6d3ee3cd064f12790534d 100644 --- a/nomad/graph/graph_reader.py +++ b/nomad/graph/graph_reader.py @@ -24,11 +24,10 @@ import functools import itertools import os import re -from collections.abc import AsyncIterator, Iterator +from collections.abc import AsyncIterator, Callable, Iterator from contextlib import contextmanager from threading import Lock from typing import Any -from collections.abc import Callable import orjson from cachetools import TTLCache @@ -55,23 +54,10 @@ from nomad.app.v1.routers.uploads import ( get_upload_with_read_access, upload_to_pydantic, ) -from nomad.archive import ( - ArchiveDict, - ArchiveList, - to_json, -) -from nomad.archive.storage_v2 import ( - ArchiveDict as ArchiveDictNew, -) -from nomad.archive.storage_v2 import ( - ArchiveList as ArchiveListNew, -) -from nomad.datamodel import ( - Dataset, - EntryArchive, - ServerContext, - User, -) +from nomad.archive import ArchiveDict, ArchiveList, to_json +from nomad.archive.storage_v2 import ArchiveDict as ArchiveDictNew +from nomad.archive.storage_v2 import ArchiveList as ArchiveListNew +from nomad.datamodel import Dataset, EntryArchive, ServerContext, User from nomad.datamodel.util import parse_path from nomad.files import RawPathInfo, UploadFiles from nomad.graph.lazy_wrapper import ( @@ -102,8 +88,8 @@ from nomad.metainfo import ( SectionReference, SubSection, ) -from nomad.metainfo.data_type import Any as AnyType from nomad.metainfo.data_type import JSON, Datatype +from nomad.metainfo.data_type import Any as AnyType from nomad.metainfo.util import MSubSectionList, split_python_definition from nomad.processing import Entry, ProcessStatus, Upload from nomad.utils import timer diff --git a/nomad/graph/lazy_wrapper.py b/nomad/graph/lazy_wrapper.py index 87b777b4833de13f0c7344454a94a0315cb25fa9..148a6c079de5944615cf1edbfd162360050392d4 100644 --- a/nomad/graph/lazy_wrapper.py +++ b/nomad/graph/lazy_wrapper.py @@ -26,6 +26,7 @@ Different wrappers are catered for different types of objects/operations. """ from __future__ import annotations + from functools import cached_property from nomad.datamodel import User diff --git a/nomad/graph/model.py b/nomad/graph/model.py index f8bf6f1c38f1150f2c4d8284d1fbeb455efd0e0f..669d7f1276e4a8386e1aadea61583917473cc7ed 100644 --- a/nomad/graph/model.py +++ b/nomad/graph/model.py @@ -25,14 +25,15 @@ from typing import Annotated, Union from pydantic import ( AfterValidator, - field_validator, - ConfigDict, BaseModel, + ConfigDict, Field, ValidationError, + field_validator, ) from nomad.app.v1.models import Direction, Metadata, MetadataPagination, Pagination +from nomad.app.v1.models.groups import UserGroupPagination, UserGroupQuery from nomad.app.v1.routers.datasets import DatasetPagination from nomad.app.v1.routers.uploads import ( EntryProcDataPagination, @@ -40,7 +41,6 @@ from nomad.app.v1.routers.uploads import ( UploadProcDataPagination, UploadProcDataQuery, ) -from nomad.app.v1.models.groups import UserGroupQuery, UserGroupPagination class DatasetQuery(BaseModel): diff --git a/nomad/infrastructure.py b/nomad/infrastructure.py index 5adbf9f90beef51ff0602410e00d50785f95812b..0d98646a160f725c7ce489d61a6d16ce8187ac95 100644 --- a/nomad/infrastructure.py +++ b/nomad/infrastructure.py @@ -23,33 +23,34 @@ is run once for each *api* and *worker* process. Individual functions for partia exist to facilitate testing, aspects of :py:mod:`nomad.cli`, etc. """ -import os.path +import json import os +import os.path +import re import shutil -from elasticsearch_dsl import connections -from mongoengine import connect, disconnect -from mongoengine.connection import ConnectionFailure import smtplib + +# TODO put somemore thought into warnings +import warnings +from datetime import datetime from email.mime.text import MIMEText -from keycloak import KeycloakOpenID, KeycloakAdmin -from keycloak.exceptions import KeycloakAuthenticationError, KeycloakGetError -import json + import jwt -from datetime import datetime -import re import unidecode +from elasticsearch_dsl import connections +from keycloak import KeycloakAdmin, KeycloakOpenID +from keycloak.exceptions import KeycloakAuthenticationError, KeycloakGetError +from mongoengine import connect, disconnect +from mongoengine.connection import ConnectionFailure from nomad import utils from nomad.config import config -from nomad.utils.structlogging import get_logger # The metainfo is defined and used during imports. This is problematic. # We import all parsers very early in the infrastructure setup. This will populate # the metainfo with parser specific definitions, before the metainfo might be used. from nomad.parsing import parsers # noqa: F401 - -# TODO put somemore thought into warnings -import warnings +from nomad.utils.structlogging import get_logger warnings.filterwarnings('ignore') diff --git a/nomad/logtransfer.py b/nomad/logtransfer.py index 6c884540c21a379be6f8eb4bbf005ef5ec207a7b..f85170d8bd3d43d93212bc83b3c9e6609bf7937b 100644 --- a/nomad/logtransfer.py +++ b/nomad/logtransfer.py @@ -16,15 +16,15 @@ # limitations under the License. # -import requests -import zlib -import os.path import os +import os.path import time +import zlib +import requests -from nomad.config import config from nomad import utils +from nomad.config import config logger = utils.get_logger(__name__) diff --git a/nomad/metainfo/annotation.py b/nomad/metainfo/annotation.py index b3bec23d57457dcf2a420fc8a68e83574ae4088c..6ee540abe928d152d28b324e2ca686c93f708f75 100644 --- a/nomad/metainfo/annotation.py +++ b/nomad/metainfo/annotation.py @@ -20,7 +20,7 @@ from __future__ import annotations from typing import Any, ClassVar, ForwardRef -from pydantic import ConfigDict, BaseModel, Field +from pydantic import BaseModel, ConfigDict, Field class Annotation: diff --git a/nomad/metainfo/data_frames.py b/nomad/metainfo/data_frames.py index 94f510e9b6952e8e940f55814702ef3ed0e745ea..4f2f1fc7d1079f32328e601cc83a30c5d7138d14 100644 --- a/nomad/metainfo/data_frames.py +++ b/nomad/metainfo/data_frames.py @@ -1,19 +1,13 @@ +import inspect import types -from typing import Union, cast from collections.abc import Iterable +from typing import Union, cast + import numpy as np -import inspect -from pydantic import BaseModel import xarray as xr +from pydantic import BaseModel -from nomad.metainfo import ( - MSection, - Section, - Quantity, - SubSection, - Package, - constraint, -) +from nomad.metainfo import MSection, Package, Quantity, Section, SubSection, constraint from nomad.metainfo.metainfo import _placeholder_quantity from nomad.units import ureg diff --git a/nomad/metainfo/data_type.py b/nomad/metainfo/data_type.py index 04b4c724651ac2a2307d9d3c55b8aba26799605b..6f9b89f798ae35a02ff7bcb57afcb6cb714d36dd 100644 --- a/nomad/metainfo/data_type.py +++ b/nomad/metainfo/data_type.py @@ -22,7 +22,7 @@ import importlib import re import typing from base64 import b64decode, b64encode -from datetime import datetime, date +from datetime import date, datetime from functools import reduce from inspect import isclass from typing import Any as TypingAny @@ -1248,12 +1248,12 @@ def to_optimade_type(in_type: Datatype): def to_mongo_type(in_type: Datatype): from mongoengine import ( - IntField, - FloatField, BooleanField, - StringField, DateTimeField, DictField, + FloatField, + IntField, + StringField, ) standard_type = in_type.standard_type() diff --git a/nomad/metainfo/elasticsearch_extension.py b/nomad/metainfo/elasticsearch_extension.py index f45f5ece73bd9d80b49030b437087563d5afef13..9d386d432962409b16d2d4f8cc34019d87cf0eb3 100644 --- a/nomad/metainfo/elasticsearch_extension.py +++ b/nomad/metainfo/elasticsearch_extension.py @@ -159,19 +159,15 @@ sub-sections as if they were direct sub-sections. import math import re from collections import defaultdict -from typing import ( - TYPE_CHECKING, - Any, - Optional, - cast, -) from collections.abc import Callable +from typing import TYPE_CHECKING, Any, Optional, cast from elasticsearch_dsl import Q +from pint import Quantity as PintQuantity + from nomad import utils from nomad.config import config from nomad.config.models.plugins import Parser, Schema, SchemaPackageEntryPoint -from pint import Quantity as PintQuantity from . import DefinitionAnnotation from .data_type import Datatype, to_elastic_type diff --git a/nomad/metainfo/example.py b/nomad/metainfo/example.py index 3e7f7443b41013722c1058de5bf70e0491629b7d..d7acce8ee9bae8c5514cce7113692b3e3663a5de 100644 --- a/nomad/metainfo/example.py +++ b/nomad/metainfo/example.py @@ -18,21 +18,22 @@ """An example metainfo package.""" -import numpy as np from datetime import datetime -from nomad.units import ureg +import numpy as np + from nomad.metainfo import ( - MSection, + Datetime, MCategory, - Section, - Quantity, + MEnum, + MSection, Package, + Quantity, + Section, SubSection, - MEnum, - Datetime, constraint, ) +from nomad.units import ureg m_package = Package(links=['https://nomad-lab.eu/prod/rae/docs/metainfo.html']) diff --git a/nomad/metainfo/metainfo.py b/nomad/metainfo/metainfo.py index 1532e571e369188ec09b1742dd25dd3f61d8a67d..67072e835dee58c21f0007d82df38f6dc6ae3e1f 100644 --- a/nomad/metainfo/metainfo.py +++ b/nomad/metainfo/metainfo.py @@ -34,7 +34,7 @@ from urllib.parse import urlsplit, urlunsplit import docstring_parser import jmespath import pint -from pydantic import TypeAdapter, ValidationError, BaseModel +from pydantic import BaseModel, TypeAdapter, ValidationError from nomad.config import config from nomad.metainfo.data_type import JSON as JSONType @@ -43,14 +43,20 @@ from nomad.metainfo.data_type import Any as AnyType from nomad.metainfo.data_type import Bytes as BytesType from nomad.metainfo.data_type import Callable as CallableType from nomad.metainfo.data_type import Capitalized as CapitalizedType -from nomad.metainfo.data_type import Datatype +from nomad.metainfo.data_type import ( + Datatype, + Enum, + ExactNumber, + InexactNumber, + Number, + check_dimensionality, + m_str, + normalize_type, +) from nomad.metainfo.data_type import Datetime as DatetimeType from nomad.metainfo.data_type import Dimension as DimensionType -from nomad.metainfo.data_type import Enum, ExactNumber from nomad.metainfo.data_type import File as FileType -from nomad.metainfo.data_type import InexactNumber, Number from nomad.metainfo.data_type import Unit as UnitType -from nomad.metainfo.data_type import check_dimensionality, m_str, normalize_type from nomad.metainfo.util import ( MQuantity, MSubSectionList, @@ -62,7 +68,6 @@ from nomad.metainfo.util import ( to_dict, ) from nomad.units import ureg as units -from pydantic import TypeAdapter, ValidationError from .annotation import ( Annotation, diff --git a/nomad/metainfo/mongoengine_extension.py b/nomad/metainfo/mongoengine_extension.py index 34c43289bf804231bc7a8e7a7bb0b87419de133d..c7dd4baa97feed7ab5921ed8e3f926708c902558 100644 --- a/nomad/metainfo/mongoengine_extension.py +++ b/nomad/metainfo/mongoengine_extension.py @@ -35,12 +35,9 @@ sections from mongoengine. The annotation key is 'mongo'. from typing import Any -from .data_type import Datatype, to_mongo_type -from .metainfo import ( - MSection, - Quantity, -) from . import Annotation, DefinitionAnnotation, SectionAnnotation +from .data_type import Datatype, to_mongo_type +from .metainfo import MSection, Quantity class Mongo(DefinitionAnnotation): diff --git a/nomad/metainfo/pydantic_extension.py b/nomad/metainfo/pydantic_extension.py index f556768aef564c43f38d0ffe5df79537cfd15af4..155140406df838c76120e987d18527833f04861f 100644 --- a/nomad/metainfo/pydantic_extension.py +++ b/nomad/metainfo/pydantic_extension.py @@ -33,15 +33,12 @@ Allows to create pydantic models from section definitions. """ from typing import cast -from pydantic import create_model, Field, BaseModel -from .data_type import to_pydantic_type -from .metainfo import ( - Definition, - Section, - Quantity, -) +from pydantic import BaseModel, Field, create_model + from . import DefinitionAnnotation +from .data_type import to_pydantic_type +from .metainfo import Definition, Quantity, Section class PydanticModel(DefinitionAnnotation): diff --git a/nomad/metainfo/util.py b/nomad/metainfo/util.py index cbaf0eb3c6f32d11bb444842afd71ff7a41e41df..a46e70d43c38bbb9eda3fb6807a3bd37af570357 100644 --- a/nomad/metainfo/util.py +++ b/nomad/metainfo/util.py @@ -22,6 +22,7 @@ import re from typing import Any import pint + from nomad.metainfo.data_type import Enum from nomad.units import ureg diff --git a/nomad/mkdocs/metainfo.py b/nomad/mkdocs/metainfo.py index 52c4ea2e3bf1f16aea0ef99a4e3b4af620a51b1a..710a6383dc7cb8ab8ae30b4a667077a7a45e49b3 100644 --- a/nomad/mkdocs/metainfo.py +++ b/nomad/mkdocs/metainfo.py @@ -15,8 +15,8 @@ # from nomad import utils -from nomad.metainfo import Property, Quantity, Datatype, Reference, SubSection from nomad.datamodel.data import ArchiveSection +from nomad.metainfo import Datatype, Property, Quantity, Reference, SubSection def get_reference(section_def, pkg) -> str: diff --git a/nomad/mkdocs/pydantic.py b/nomad/mkdocs/pydantic.py index eb55aa89b76ebc3c16cfa55e1545b34be41c70ac..774843aa3d48ee149cdda56f32517b3e9543fd54 100644 --- a/nomad/mkdocs/pydantic.py +++ b/nomad/mkdocs/pydantic.py @@ -20,19 +20,16 @@ Definitions that are used in the documentation via mkdocs-macro-plugin. """ +from enum import Enum +from inspect import isclass from types import UnionType -from pydantic.fields import FieldInfo +from typing import Annotated, Any, Literal, Union, get_args, get_origin -from enum import Enum from pydantic import BaseModel - -from typing import Annotated, Any, Union, get_args, get_origin -from typing import Literal -from inspect import isclass +from pydantic.fields import FieldInfo from nomad import utils - exported_config_models = set() # type: ignore diff --git a/nomad/normalizing/common.py b/nomad/normalizing/common.py index 9075f97a704174bfaacb5662aa1ef225ae23a2ba..f03185579dc8dbdbec5a0075841fa9f115be9ade 100644 --- a/nomad/normalizing/common.py +++ b/nomad/normalizing/common.py @@ -15,26 +15,24 @@ # See the License for the specific language governing permissions and # limitations under the License. # +from typing import Any + +import matid.geometry # pylint: disable=import-error import numpy as np from ase import Atoms -from typing import Any -from nptyping import NDArray from matid import SymmetryAnalyzer # pylint: disable=import-error -from matid.symmetry.wyckoffset import WyckoffSet as WyckoffSetMatID # pylint: disable=import-error -import matid.geometry # pylint: disable=import-error +from matid.symmetry.wyckoffset import ( + WyckoffSet as WyckoffSetMatID, # pylint: disable=import-error +) +from nptyping import NDArray from nomad import atomutils from nomad.config import config -from nomad.utils import hash -from nomad.units import ureg from nomad.datamodel.metainfo.system import Atoms as NOMADAtoms from nomad.datamodel.optimade import Species -from nomad.datamodel.results import ( - Cell, - Structure, - LatticeParameters, - WyckoffSet, -) +from nomad.datamodel.results import Cell, LatticeParameters, Structure, WyckoffSet +from nomad.units import ureg +from nomad.utils import hash def wyckoff_sets_from_matid(wyckoff_sets: list[WyckoffSetMatID]) -> list[WyckoffSet]: diff --git a/nomad/normalizing/material.py b/nomad/normalizing/material.py index f5eed863aa88ba8ed2c8790dfd673e74de7e6de0..9192162c68cabba23ff671779e183f158afb39b6 100644 --- a/nomad/normalizing/material.py +++ b/nomad/normalizing/material.py @@ -16,24 +16,24 @@ # import re -from nptyping import NDArray import ase.data from matid.classification.classifications import ( - Class0D, Atom, + Class0D, Class1D, Class2D, + Class3D, Material2D, Surface, - Class3D, ) +from nptyping import NDArray from nomad import atomutils from nomad.atomutils import Formula -from nomad.normalizing.common import material_id_bulk, material_id_2d, material_id_1d +from nomad.datamodel.results import Material, Symmetry, structure_name_map +from nomad.normalizing.common import material_id_1d, material_id_2d, material_id_bulk from nomad.normalizing.topology import TopologyNormalizer -from nomad.datamodel.results import Symmetry, Material, structure_name_map class MaterialNormalizer: diff --git a/nomad/normalizing/metainfo.py b/nomad/normalizing/metainfo.py index 11238e9224a839a1d4e538c90343ae695bfa18d9..73e3ddd01829687edab628a8065c73107b60fe26 100644 --- a/nomad/normalizing/metainfo.py +++ b/nomad/normalizing/metainfo.py @@ -18,7 +18,7 @@ from nomad.datamodel import EntryArchive from nomad.datamodel.data import ArchiveSection -from nomad.datamodel import EntryArchive + from . import Normalizer diff --git a/nomad/normalizing/method.py b/nomad/normalizing/method.py index 0fc961ec59be8499fe209547351a58d40abe2f8b..2a3355f8b13bf6b9c87d5be54b4f0c5aa54011e1 100644 --- a/nomad/normalizing/method.py +++ b/nomad/normalizing/method.py @@ -15,33 +15,33 @@ # See the License for the specific language governing permissions and # limitations under the License. # +import re from abc import ABC, abstractmethod -from ase.dft.kpoints import monkhorst_pack, get_monkhorst_pack_size_and_offset from collections import OrderedDict -import re + import numpy as np +from ase.dft.kpoints import get_monkhorst_pack_size_and_offset, monkhorst_pack -from nomad.datamodel import EntryArchive, ArchiveSection -from nomad.metainfo import MSection -from nomad.metainfo.data_type import Number -from nomad.units import ureg -from nomad.metainfo import Section -from nomad.utils import RestrictedDict from nomad.config import config +from nomad.datamodel import ArchiveSection, EntryArchive from nomad.datamodel.results import ( - Method, - Simulation, - HubbardKanamoriModel, - DFT, - TB, - GW, BSE, + DFT, DMFT, - Precision, + GW, + TB, + HubbardKanamoriModel, Material, + Method, + Precision, + Simulation, xc_treatments, xc_treatments_extended, ) +from nomad.metainfo import MSection, Section +from nomad.metainfo.data_type import Number +from nomad.units import ureg +from nomad.utils import RestrictedDict class MethodNormalizer: # TODO: add normalizer for atom_parameters.label diff --git a/nomad/normalizing/normalizer.py b/nomad/normalizing/normalizer.py index 320571ccbae420d69c1ccbd3a23aedf4c28bf3d2..0c3976f29788071f990c8c94349cacf36134cc81 100644 --- a/nomad/normalizing/normalizer.py +++ b/nomad/normalizing/normalizer.py @@ -18,9 +18,9 @@ from abc import ABCMeta, abstractmethod -from nomad.utils import get_logger -from nomad.metainfo import MSection from nomad.datamodel import EntryArchive +from nomad.metainfo import MSection +from nomad.utils import get_logger class Normalizer(metaclass=ABCMeta): diff --git a/nomad/normalizing/optimade.py b/nomad/normalizing/optimade.py index f6e1913e46d6d7afef2d40d5e9c16272d5172065..3c78ab4ca58561576318bd6507cce93685270a8e 100644 --- a/nomad/normalizing/optimade.py +++ b/nomad/normalizing/optimade.py @@ -16,19 +16,18 @@ # limitations under the License. # -from typing import Any -import numpy as np import re +from typing import Any + import ase.data import ase.formula +import numpy as np import pint.quantity -from nomad.datamodel import EntryArchive from nomad.atomutils import Formula +from nomad.datamodel import EntryArchive, EntryMetadata, OptimadeEntry, Species from nomad.normalizing.normalizer import SystemBasedNormalizer from nomad.units import ureg -from nomad.datamodel import OptimadeEntry, Species, EntryMetadata - species_re = re.compile(r'^([A-Z][a-z]?)(\d*)$') atom_label_re = re.compile( diff --git a/nomad/normalizing/results.py b/nomad/normalizing/results.py index 85657aa40696c203ea4f667020b439e34ac6bb99..2316cb4510002da124d5deb5408224a3354218ea 100644 --- a/nomad/normalizing/results.py +++ b/nomad/normalizing/results.py @@ -17,77 +17,78 @@ # import re -import numpy as np from typing import Any + import ase.data -from matid import SymmetryAnalyzer # pylint: disable=import-error import matid.geometry # pylint: disable=import-error +import numpy as np +from matid import SymmetryAnalyzer # pylint: disable=import-error from nomad import atomutils -from nomad.config import config -from nomad.utils import traverse_reversed, extract_section from nomad.atomutils import Formula -from nomad.normalizing.normalizer import Normalizer -from nomad.normalizing.method import MethodNormalizer -from nomad.normalizing.material import MaterialNormalizer +from nomad.config import config from nomad.datamodel import EntryArchive -from nomad.datamodel.metainfo.workflow import Workflow from nomad.datamodel.data import ArchiveSection -from nomad.normalizing.common import structures_2d +from nomad.datamodel.metainfo.workflow import Workflow from nomad.datamodel.results import ( BandGap, BandGapDeprecated, - RadialDistributionFunction, - RadiusOfGyration, - MeanSquaredDisplacement, - Results, - Material, - Method, - GeometryOptimization, - Trajectory, - MolecularDynamics, - MDProvenance, - TemperatureDynamic, - VolumeDynamic, - PressureDynamic, - EnergyDynamic, - Properties, - StructuralProperties, - DynamicalProperties, - EnergyVolumeCurve, - BulkModulus, - ShearModulus, - MechanicalProperties, - ElectronicProperties, - VibrationalProperties, - ThermodynamicProperties, BandStructureElectronic, BandStructurePhonon, + BulkModulus, + DensityCharge, DOSElectronic, - DOSNew, DOSElectronicNew, + DOSNew, DOSPhonon, - GreensFunctionsElectronic, + DynamicalProperties, + EELSMethodology, + ElectricFieldGradient, + ElectronicProperties, + EnergyDynamic, EnergyFreeHelmholtz, + EnergyVolumeCurve, + GeometryOptimization, + GreensFunctionsElectronic, HeatCapacityConstantVolume, - SpectroscopicProperties, - EELSMethodology, - SpectraProvenance, - Spectra, MagneticProperties, MagneticShielding, MagneticSusceptibility, - ElectricFieldGradient, + Material, + MDProvenance, + MeanSquaredDisplacement, + MechanicalProperties, + Method, + MolecularDynamics, + PressureDynamic, + Properties, + RadialDistributionFunction, + RadiusOfGyration, + Results, + ShearModulus, + Spectra, + SpectraProvenance, + SpectroscopicProperties, SpinSpinCoupling, - DensityCharge, + StructuralProperties, + TemperatureDynamic, + ThermodynamicProperties, + Trajectory, + VibrationalProperties, + VolumeDynamic, ) +from nomad.normalizing.common import structures_2d +from nomad.normalizing.material import MaterialNormalizer +from nomad.normalizing.method import MethodNormalizer +from nomad.normalizing.normalizer import Normalizer +from nomad.utils import extract_section, traverse_reversed try: import runschema runschema.run_schema_entry_point.load() - import runschema.method import runschema.calculation + import runschema.method import runschema.system except Exception as e: runschema, simulationworkflowschema = None, None diff --git a/nomad/normalizing/topology.py b/nomad/normalizing/topology.py index 3a6f4a22e4ee5e84d051b5f7f89485ffdd5c71dc..2477f91ad67dbb7dd591002a481001838c511919 100644 --- a/nomad/normalizing/topology.py +++ b/nomad/normalizing/topology.py @@ -16,46 +16,45 @@ # limitations under the License. # -from collections import defaultdict -import pathlib import json +import pathlib +from collections import defaultdict +import numpy as np from ase import Atoms from ase.data import chemical_symbols -import numpy as np -from matid.clustering import SBC, Cluster -from matid.symmetry.symmetryanalyzer import SymmetryAnalyzer from matid.classification.classifications import ( - Class0D, Atom, + Class0D, Class1D, Class2D, + Class3D, Material2D, Surface, - Class3D, ) +from matid.clustering import SBC, Cluster +from matid.symmetry.symmetryanalyzer import SymmetryAnalyzer -from nomad import utils +from nomad import atomutils, utils from nomad.config import config -from nomad import atomutils +from nomad.datamodel.datamodel import EntryArchive from nomad.datamodel.results import ( CoreHole, - SymmetryNew as Symmetry, Material, - System, Relation, + System, structure_name_map, ) -from nomad.datamodel.datamodel import EntryArchive +from nomad.datamodel.results import SymmetryNew as Symmetry from nomad.normalizing.common import ( - cell_from_ase_atoms, ase_atoms_from_nomad_atoms, + cell_from_ase_atoms, + material_id_1d, + material_id_2d, + material_id_bulk, nomad_atoms_from_ase_atoms, - wyckoff_sets_from_matid, structures_2d, - material_id_bulk, - material_id_2d, - material_id_1d, + wyckoff_sets_from_matid, ) conventional_description = 'The conventional cell of the material from which the subsystem is constructed from.' diff --git a/nomad/parsing/artificial.py b/nomad/parsing/artificial.py index 6302b72dde31231c95de7924a889b9d38d20423e..ae4f1b74b182c8bda27412689866722071b53fbd 100644 --- a/nomad/parsing/artificial.py +++ b/nomad/parsing/artificial.py @@ -21,19 +21,20 @@ Parser for creating artificial test, brenchmark, and demonstration data. """ import json +import os import os.path import random -from ase.data import chemical_symbols -import numpy +import signal import sys import time -import os -import signal + +import numpy +from ase.data import chemical_symbols from nomad.datamodel import EntryArchive from nomad.datamodel.metainfo import runschema -from .parser import Parser, MatchingParser +from .parser import MatchingParser, Parser class EmptyParser(MatchingParser): diff --git a/nomad/parsing/file_parser/file_parser.py b/nomad/parsing/file_parser/file_parser.py index f66274b69b1c638d585f26926ec4f2a2979183f6..b9bef402991694ccb5fa5b64d5d25f5eedecbeb4 100644 --- a/nomad/parsing/file_parser/file_parser.py +++ b/nomad/parsing/file_parser/file_parser.py @@ -12,20 +12,21 @@ # See the License for the specific language governing permissions and # limitations under the License. -from abc import ABC, abstractmethod -import os -import pint -from typing import Any, IO -from collections.abc import Callable -import gzip import bz2 +import gzip import lzma +import os import tarfile +from abc import ABC, abstractmethod +from collections.abc import Callable from contextlib import contextmanager +from typing import IO, Any + +import pint +from nomad.datamodel import EntryArchive from nomad.metainfo import MSection, SubSection from nomad.utils import get_logger -from nomad.datamodel import EntryArchive class FileParser(ABC): diff --git a/nomad/parsing/file_parser/mapping_parser.py b/nomad/parsing/file_parser/mapping_parser.py index d443d93fc0207ee0e6968f0091c3ee3cf1b6bc9a..5d0be33bcab33d614518f8d1528a47c37a635750 100644 --- a/nomad/parsing/file_parser/mapping_parser.py +++ b/nomad/parsing/file_parser/mapping_parser.py @@ -11,6 +11,8 @@ import jmespath.visitor import numpy as np from jsonpath_ng.parser import JsonPathParser from lxml import etree +from pydantic import BaseModel, Field, model_validator + from nomad.datamodel import EntryArchive from nomad.datamodel.metainfo.annotations import Mapper as MapperAnnotation from nomad.metainfo import MSection, SubSection @@ -18,7 +20,6 @@ from nomad.parsing.file_parser import TextParser as TextFileParser from nomad.parsing.parser import ArchiveParser from nomad.units import ureg from nomad.utils import get_logger -from pydantic import BaseModel, Field, model_validator MAPPING_ANNOTATION_KEY = 'mapping' diff --git a/nomad/parsing/file_parser/text_parser.py b/nomad/parsing/file_parser/text_parser.py index 5010ee16736025e94a6d5e8f4abce69bcf2856c1..a742e171ebae728ad626fd92ec69f010acd41f39 100644 --- a/nomad/parsing/file_parser/text_parser.py +++ b/nomad/parsing/file_parser/text_parser.py @@ -13,16 +13,17 @@ # limitations under the License. -import mmap import io +import mmap import re +from collections.abc import Callable +from typing import Any + import numpy as np import pint -from typing import Any -from collections.abc import Callable -from nomad.parsing.file_parser import FileParser from nomad.metainfo import Quantity as mQuantity +from nomad.parsing.file_parser import FileParser from nomad.utils import get_logger diff --git a/nomad/parsing/file_parser/xml_parser.py b/nomad/parsing/file_parser/xml_parser.py index e1fecc6b3f32a359ad4c3ca96806cd56731ab4d8..5739ec9bf394c42c376a4a2fcd82de38437494dc 100644 --- a/nomad/parsing/file_parser/xml_parser.py +++ b/nomad/parsing/file_parser/xml_parser.py @@ -15,8 +15,9 @@ import os import re -import numpy as np from xml.etree import ElementTree + +import numpy as np from lxml import etree from nomad.parsing.file_parser import FileParser diff --git a/nomad/parsing/parser.py b/nomad/parsing/parser.py index b8882362997506f89dbe900330f22e7805afc6c5..9dd04e06fd0d6f699571370befde56b6fecb26dd 100644 --- a/nomad/parsing/parser.py +++ b/nomad/parsing/parser.py @@ -16,19 +16,20 @@ # limitations under the License. # -from typing import Any, IO -from collections.abc import Iterable -from abc import ABCMeta, abstractmethod -import re +import importlib +import json import os import os.path +import re +from abc import ABCMeta, abstractmethod +from collections.abc import Iterable from functools import lru_cache -import importlib -from pydantic import BaseModel, Extra # noqa: F401 -import yaml +from typing import IO, Any + import h5py import numpy as np -import json +import yaml +from pydantic import BaseModel, Extra # noqa: F401 from nomad import utils from nomad.config import config diff --git a/nomad/parsing/parsers.py b/nomad/parsing/parsers.py index f6b5acdd5494617f9482aaa94015c6b97b93e114..47b3653da34dc7836ffe9823189a033f536d71ce 100644 --- a/nomad/parsing/parsers.py +++ b/nomad/parsing/parsers.py @@ -20,28 +20,30 @@ import os.path from collections.abc import Iterable from nomad.config import config -from nomad.config.models.plugins import Parser as ParserPlugin, ParserEntryPoint +from nomad.config.models.plugins import Parser as ParserPlugin +from nomad.config.models.plugins import ParserEntryPoint from nomad.datamodel import EntryArchive, EntryMetadata, results -from nomad.datamodel.context import Context, ClientContext +from nomad.datamodel.context import ClientContext, Context +from .artificial import ChaosParser, EmptyParser, GenerateRandomParser, TemplateParser from .parser import ( - MissingParser, - BrokenParser, - Parser, ArchiveParser, + BrokenParser, MatchingParserInterface, + MissingParser, + Parser, ) -from .artificial import EmptyParser, GenerateRandomParser, TemplateParser, ChaosParser from .tabular import TabularDataParser try: # these packages are not available without parsing extra, which is ok, if the # parsers are only initialized to load their metainfo definitions - import magic - import gzip import bz2 + import gzip import lzma + import magic + _compressions = { b'\x1f\x8b\x08': ('gz', gzip.open), b'\x42\x5a\x68': ('bz2', bz2.open), diff --git a/nomad/parsing/tabular.py b/nomad/parsing/tabular.py index ebfd2a873d158662a91f69429d0852fcf45e7d0e..38082a6d814ba919501bdb945d90823910d586a1 100644 --- a/nomad/parsing/tabular.py +++ b/nomad/parsing/tabular.py @@ -15,32 +15,31 @@ # See the License for the specific language governing permissions and # limitations under the License. # +import json +import math import os +import re +from collections.abc import Callable, Iterable, Iterator from typing import Any -from collections.abc import Callable -from collections.abc import Iterator, Iterable -import pandas as pd -import re -import math import numpy as np -import json +import pandas as pd import yaml -from cachetools import cached, LRUCache +from cachetools import LRUCache, cached from nomad import utils -from nomad.parsing import MatchingParser -from nomad.units import ureg from nomad.datamodel.data import ArchiveSection -from nomad.metainfo import Section, Quantity, Package, Reference, MSection, Property -from nomad.metainfo.metainfo import MetainfoError, SubSection, MProxy from nomad.datamodel.metainfo.annotations import ( TabularAnnotation, - TabularParserAnnotation, TabularFileModeEnum, TabularMode, + TabularParserAnnotation, ) +from nomad.metainfo import MSection, Package, Property, Quantity, Reference, Section +from nomad.metainfo.metainfo import MetainfoError, MProxy, SubSection from nomad.metainfo.util import MSubSectionList +from nomad.parsing import MatchingParser +from nomad.units import ureg from nomad.utils import generate_entry_id # We define a simple base schema for tabular data. The parser will then generate more diff --git a/nomad/processing/base.py b/nomad/processing/base.py index 17a6d7d6821e2d534c282331241057d68b937d2c..40a93ce18ee9eae21f113746ca5307e6322684db 100644 --- a/nomad/processing/base.py +++ b/nomad/processing/base.py @@ -16,42 +16,42 @@ # limitations under the License. # -from typing import Any, NamedTuple +import functools import logging -import time import os +import time from collections import defaultdict +from datetime import datetime +from typing import Any, NamedTuple + +import billiard +from billiard.exceptions import WorkerLostError from celery import Celery, Task -from celery.worker.request import Request +from celery.exceptions import SoftTimeLimitExceeded from celery.signals import ( - after_setup_task_logger, after_setup_logger, - worker_process_init, + after_setup_task_logger, celeryd_after_setup, + worker_process_init, worker_process_shutdown, ) from celery.utils import worker_direct -from celery.exceptions import SoftTimeLimitExceeded -import billiard -from billiard.exceptions import WorkerLostError +from celery.worker.request import Request from mongoengine import ( - Document, - StringField, - ListField, DateTimeField, + Document, IntField, + ListField, + StringField, ValidationError, ) from mongoengine.connection import ConnectionFailure -from datetime import datetime -import functools -from nomad import utils, infrastructure +import nomad.patch # noqa: F401 +from nomad import infrastructure, utils +from nomad.app.v1.routers.info import statistics from nomad.config import config from nomad.config.models.config import CELERY_WORKER_ROUTING -from nomad.app.v1.routers.info import statistics -import nomad.patch # noqa: F401 - if config.logstash.enabled: from nomad.utils import structlogging diff --git a/nomad/processing/data.py b/nomad/processing/data.py index b75902e2c3391da3d8d6700205ab1ec25bdb377a..706ebd80ebe3c3ff45b89cce651efac314d19bb0 100644 --- a/nomad/processing/data.py +++ b/nomad/processing/data.py @@ -29,96 +29,84 @@ entries, and files """ import base64 -from typing import ( - cast, - Any, - Union, -) -from collections.abc import Iterator, Iterable, Sequence +import copy +import hashlib +import os.path +from collections.abc import Iterable, Iterator, Sequence +from contextlib import contextmanager +from datetime import datetime +from typing import Any, Union, cast + +import requests import rfc3161ng +import validators +from fastapi.exceptions import RequestValidationError from mongoengine import ( - StringField, - DateTimeField, BooleanField, - IntField, - ListField, + DateTimeField, DictField, EmbeddedDocument, EmbeddedDocumentField, + IntField, + ListField, + StringField, ) from pymongo import UpdateOne from structlog import wrap_logger -from contextlib import contextmanager -import copy -import os.path -from datetime import datetime -import hashlib -from structlog.processors import StackInfoRenderer, format_exc_info, TimeStamper -import requests -from fastapi.exceptions import RequestValidationError -import validators +from structlog.processors import StackInfoRenderer, TimeStamper, format_exc_info -from nomad import ( - utils, - infrastructure, - search, - datamodel, - metainfo, - parsing, - client, +from nomad import client, datamodel, infrastructure, metainfo, parsing, search, utils +from nomad.app.v1.models import ( + Aggregation, + MetadataEditRequest, + MetadataPagination, + MetadataRequired, + TermsAggregation, + restrict_query_to_upload, +) +from nomad.app.v1.routers.metainfo import store_package_definition +from nomad.archive import ( + delete_partial_archives_from_mongo, + to_json, + write_partial_archive_to_mongo, ) -from nomad.config import config from nomad.common import is_safe_relative_path +from nomad.config import config +from nomad.config.models.config import Reprocess from nomad.config.models.plugins import ExampleUploadEntryPoint - +from nomad.datamodel import ( + AuthLevel, + EditableUserMetadata, + EntryArchive, + EntryMetadata, + MongoEntryMetadata, + MongoSystemMetadata, + MongoUploadMetadata, + ServerContext, +) from nomad.datamodel.datamodel import RFC3161Timestamp from nomad.files import ( - RawPathInfo, PathObject, - UploadFiles, PublicUploadFiles, + RawPathInfo, StagingUploadFiles, + UploadFiles, create_tmp_dir, ) -from nomad.groups import user_group_exists, get_group_ids +from nomad.groups import get_group_ids, user_group_exists from nomad.metainfo.data_type import Datatype, Datetime +from nomad.normalizing import normalizers +from nomad.parsing import Parser +from nomad.parsing.parsers import match_parser, parser_dict from nomad.processing.base import ( Proc, + ProcessAlreadyRunning, + ProcessFailure, + ProcessStatus, process, process_local, - ProcessStatus, - ProcessFailure, - ProcessAlreadyRunning, -) -from nomad.parsing import Parser -from nomad.parsing.parsers import parser_dict, match_parser -from nomad.normalizing import normalizers -from nomad.datamodel import ( - EntryArchive, - EntryMetadata, - MongoUploadMetadata, - MongoEntryMetadata, - MongoSystemMetadata, - EditableUserMetadata, - AuthLevel, - ServerContext, ) -from nomad.archive import ( - write_partial_archive_to_mongo, - delete_partial_archives_from_mongo, - to_json, -) -from nomad.app.v1.models import ( - MetadataEditRequest, - Aggregation, - TermsAggregation, - MetadataPagination, - MetadataRequired, - restrict_query_to_upload, -) -from nomad.app.v1.routers.metainfo import store_package_definition from nomad.search import update_metadata as es_update_metadata -from nomad.config.models.config import Reprocess from nomad.utils.pydantic import CustomErrorWrapper section_metadata = datamodel.EntryArchive.metadata.name diff --git a/nomad/search.py b/nomad/search.py index 52b8a21d85b65d513a4bcad67c2c568a1f81d5f4..0a3785622d88504bdd5733db27ae3134972d54ae 100644 --- a/nomad/search.py +++ b/nomad/search.py @@ -34,13 +34,9 @@ partially implemented. import json import math +from collections.abc import Callable, Generator, Iterable, Iterator from enum import Enum -from typing import ( - Any, - cast, -) -from collections.abc import Callable -from collections.abc import Generator, Iterable, Iterator +from typing import Any, cast import elasticsearch.helpers from elasticsearch.exceptions import RequestError, TransportError @@ -80,12 +76,7 @@ from nomad.app.v1.models.models import ( Value, ) from nomad.config import config -from nomad.datamodel import ( - EntryArchive, - EntryMetadata, - AuthorReference, - UserReference, -) +from nomad.datamodel import AuthorReference, EntryArchive, EntryMetadata, UserReference from nomad.groups import MongoUserGroup from nomad.metainfo import Datetime, Package, Quantity from nomad.metainfo.elasticsearch_extension import ( diff --git a/nomad/utils/exampledata.py b/nomad/utils/exampledata.py index 7846e8f9fe99aef0a3064ede2d5fbd3bf5b2db3e..4603a39b65b32d6c9e5afa016c34e1ed1275981d 100644 --- a/nomad/utils/exampledata.py +++ b/nomad/utils/exampledata.py @@ -16,16 +16,16 @@ # limitations under the License. # -from typing import Any -from datetime import datetime, timedelta import os +from datetime import datetime, timedelta +from typing import Any -from nomad import search, files -from nomad.datamodel import EntryMetadata, EntryArchive, Results +from nomad import files, search +from nomad.datamodel import EntryArchive, EntryMetadata, Results +from nomad.datamodel.metainfo import runschema from nomad.datamodel.metainfo.workflow import Workflow -from nomad.processing.data import mongo_upload_metadata from nomad.normalizing import normalizers -from nomad.datamodel.metainfo import runschema +from nomad.processing.data import mongo_upload_metadata class ExampleData: @@ -61,8 +61,8 @@ class ExampleData: es_nomad_version: str = None, archive_nomad_version: str = None, ): - from tests.test_files import create_test_upload_files from nomad import processing as proc + from tests.test_files import create_test_upload_files errors = None @@ -145,8 +145,8 @@ class ExampleData: parser_name: str | None = None, ): """Creates an entry from a mainfile which then gets parsed and normalized.""" - from nomad.parsing import parsers from nomad import parsing + from nomad.parsing import parsers assert upload_id in self.uploads, 'Must create the upload first' diff --git a/nomad/utils/json_transformer.py b/nomad/utils/json_transformer.py index 4d07d685371147fc33757916abc611e8afc42254..1a59ace7925db142aa3ab2849de82dd2272890b4 100644 --- a/nomad/utils/json_transformer.py +++ b/nomad/utils/json_transformer.py @@ -20,7 +20,7 @@ from typing import Any import jmespath -from nomad.datamodel.metainfo.annotations import Rules, Rule, Condition +from nomad.datamodel.metainfo.annotations import Condition, Rule, Rules class Transformer: diff --git a/nomad/utils/structlogging.py b/nomad/utils/structlogging.py index f0303ee1b55cdc05211bdfdb52809a4fd8d8e427..3cd4d85ba4e90b837d2be650e1cb8c6d9420d017 100644 --- a/nomad/utils/structlogging.py +++ b/nomad/utils/structlogging.py @@ -25,21 +25,22 @@ take keyword arguments for structured data. Otherwise `get_logger` can be used similar to the standard `logging.getLogger`. """ -from typing import cast, Any +import json import logging +import os.path +import re from logging.handlers import WatchedFileHandler +from typing import Any, cast + +import logstash import structlog from structlog.processors import ( + JSONRenderer, StackInfoRenderer, - format_exc_info, TimeStamper, - JSONRenderer, + format_exc_info, ) from structlog.stdlib import LoggerFactory -import logstash -import json -import re -import os.path from nomad import utils from nomad.config import config diff --git a/pyproject.toml b/pyproject.toml index 698b09fa88a9a80f465903ccda0607ac6755d448..a51ecf825cdcd9d828d9370f63acb2b6b405bd38 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -154,6 +154,7 @@ indent-width = 4 select = [ "E", # pycodestyle "F401", # remove unused import + "I001", # sort imports "PL", # pylint "UP", # pyupgrade "W", # pycodestyle @@ -174,9 +175,10 @@ ignore = [ "PLR5501", # else-if-used ] fixable = ["ALL"] +isort.split-on-trailing-comma = false [tool.ruff.lint.extend-per-file-ignores] -"__init__.py" = ["F401"] +"__init__.py" = ["F401", "I001"] "nomad/app/v1/models/graph/utils.py" = [ "UP007", ] # causes pydantic model building errors diff --git a/tests/app/test_app.py b/tests/app/test_app.py index 12a36607c0894240027972d8c5fabd4664d73e00..6140f085b980fbecd8dbbd1f1ce20ebff0759611 100644 --- a/tests/app/test_app.py +++ b/tests/app/test_app.py @@ -16,9 +16,10 @@ # limitations under the License. # -import pytest import os +import pytest + from nomad.config import config diff --git a/tests/app/test_dcat.py b/tests/app/test_dcat.py index 4f08b698f2a66cf80bed6f094aaaa07989a98cb2..1f0ecd711152a3b32edbf0e60731fe4e04016f3e 100644 --- a/tests/app/test_dcat.py +++ b/tests/app/test_dcat.py @@ -16,14 +16,15 @@ # limitations under the License. # -import pytest from datetime import datetime + +import pytest from fastapi.testclient import TestClient from nomad.app.dcat.main import app from nomad.app.dcat.mapping import Mapping -from nomad.datamodel.results import Material, Results from nomad.datamodel import Dataset +from nomad.datamodel.results import Material, Results from nomad.utils.exampledata import ExampleData diff --git a/tests/app/test_h5grove.py b/tests/app/test_h5grove.py index 592cb623ec0faca0212bb4fe56f70f17a8620f2c..a5b7446b381109205ee1799ba17acbb10e3aaf30 100644 --- a/tests/app/test_h5grove.py +++ b/tests/app/test_h5grove.py @@ -15,15 +15,16 @@ # See the License for the specific language governing permissions and # limitations under the License. # -import pytest import os + import h5py +import pytest from fastapi.testclient import TestClient from nomad.app import h5grove_app -from nomad.utils.exampledata import ExampleData -from nomad.files import StagingUploadFiles from nomad.config import config +from nomad.files import StagingUploadFiles +from nomad.utils.exampledata import ExampleData @pytest.fixture diff --git a/tests/app/test_optimade.py b/tests/app/test_optimade.py index a7841ff89a7a45e7ca0c8977f932047c5dab4623..3b95f9173ba0f0cfad64ecda1cad549457118c8e 100644 --- a/tests/app/test_optimade.py +++ b/tests/app/test_optimade.py @@ -17,15 +17,15 @@ # import json + import pytest -from nomad.processing import Upload from nomad import utils -from nomad.search import search from nomad.app.optimade import parse_filter from nomad.app.optimade.common import provider_specific_fields +from nomad.processing import Upload +from nomad.search import search from nomad.utils.exampledata import ExampleData - from tests.fixtures.infrastructure import clear_elastic, clear_raw_files diff --git a/tests/app/test_resources.py b/tests/app/test_resources.py index e76d787079aabe24883b2fa234e448f52ee16363..4f1b3929fa0e0bcd263241dd16877d77cc56cfcc 100644 --- a/tests/app/test_resources.py +++ b/tests/app/test_resources.py @@ -15,21 +15,22 @@ # See the License for the specific language governing permissions and # limitations under the License. # -import pytest import json -from fastapi.testclient import TestClient -import httpx -from urllib.parse import urlencode import time +from urllib.parse import urlencode + import dateutil.parser +import httpx +import pytest +from fastapi.testclient import TestClient -from nomad.config import config from nomad.app.resources.main import app, remove_mongo from nomad.app.resources.routers.resources import ( aflow_prototypes_db, - springer_materials_db, optimade_providers, + springer_materials_db, ) +from nomad.config import config def _to_datetime(datetime_str): diff --git a/tests/app/v1/routers/common.py b/tests/app/v1/routers/common.py index 55a522a635e37c435cfb0fcc0eafec66a80fce26..20f17289ddb49fa9e1b8a23161ce85ada5f1fafe 100644 --- a/tests/app/v1/routers/common.py +++ b/tests/app/v1/routers/common.py @@ -16,17 +16,16 @@ # limitations under the License. # -import pytest -from typing import Literal, Any import json import re -from devtools import debug +from typing import Any, Literal from urllib.parse import urlencode -from nomad.utils import deep_get +import pytest +from devtools import debug from nomad.datamodel import results - +from nomad.utils import deep_get from tests.utils import assert_at_least, assert_url_query_args, build_url n_code_names = results.Simulation.program_name.a_elasticsearch[ diff --git a/tests/app/v1/routers/test_auth.py b/tests/app/v1/routers/test_auth.py index c814260a3c4b502c8c841b09a2cb1fd2eb1302fe..2c5e3e67c8f42da033bc0c1bc716025638f89d63 100644 --- a/tests/app/v1/routers/test_auth.py +++ b/tests/app/v1/routers/test_auth.py @@ -16,9 +16,10 @@ # limitations under the License. # -import pytest from urllib.parse import urlencode +import pytest + def perform_get_token_test(client, http_method, status_code, username, password): if http_method == 'post': diff --git a/tests/app/v1/routers/test_datasets.py b/tests/app/v1/routers/test_datasets.py index 36cd3de91c9161be75af1930c91b5a729102d73e..7153ee770a9b3a076f6f7072354924a2d0f7deff 100644 --- a/tests/app/v1/routers/test_datasets.py +++ b/tests/app/v1/routers/test_datasets.py @@ -16,16 +16,16 @@ # limitations under the License. # -import pytest -from urllib.parse import urlencode from datetime import datetime +from urllib.parse import urlencode + +import pytest -from nomad.datamodel import Dataset from nomad import processing +from nomad.app.v1.models import Any_, Query +from nomad.datamodel import Dataset from nomad.search import search -from nomad.app.v1.models import Query, Any_ from nomad.utils.exampledata import ExampleData - from tests.fixtures.users import admin_user_id from .common import assert_response diff --git a/tests/app/v1/routers/test_entries.py b/tests/app/v1/routers/test_entries.py index fbdfaaae175e5ccf8289ba8a3f503aca1df93711..36985eb08f55098d5930560ca4707427e14f710a 100644 --- a/tests/app/v1/routers/test_entries.py +++ b/tests/app/v1/routers/test_entries.py @@ -16,37 +16,37 @@ # limitations under the License. # -import pytest -from urllib.parse import urlencode -import zipfile import io import json +import zipfile +from urllib.parse import urlencode + +import pytest from nomad.metainfo.elasticsearch_extension import entry_type, schema_separator from nomad.utils.exampledata import ExampleData - -from tests.test_files import example_mainfile_contents, append_raw_files # noqa: F401 +from tests.test_files import append_raw_files, example_mainfile_contents # noqa: F401 from tests.variables import python_schema_name from .common import ( aggregation_exclude_from_search_test_parameters, - assert_response, + aggregation_test_parameters, + aggregation_test_parameters_default, + assert_aggregation_response, + assert_aggregations, assert_base_metadata_response, - assert_query_response, + assert_browser_download_headers, assert_metadata_response, - assert_required, - assert_aggregations, assert_pagination, - assert_browser_download_headers, - post_query_test_parameters, + assert_query_response, + assert_required, + assert_response, get_query_test_parameters, - perform_owner_test, owner_test_parameters, pagination_test_parameters, - aggregation_test_parameters, - aggregation_test_parameters_default, - assert_aggregation_response, perform_entries_metadata_test, + perform_owner_test, + post_query_test_parameters, ) """ diff --git a/tests/app/v1/routers/test_entries_archive_edit.py b/tests/app/v1/routers/test_entries_archive_edit.py index 8a2e90107dbffd9b62c3d1d9677d5627e4ed7559..ba9903ebde1018599934eccc59a914e1243c6f9b 100644 --- a/tests/app/v1/routers/test_entries_archive_edit.py +++ b/tests/app/v1/routers/test_entries_archive_edit.py @@ -17,6 +17,7 @@ # import json + import pytest from nomad.datamodel.datamodel import EntryArchive, EntryMetadata diff --git a/tests/app/v1/routers/test_entries_edit.py b/tests/app/v1/routers/test_entries_edit.py index cd793d3fb4c43a5696bf9dc7800296de9b254b88..1d3db4a4df67d2a0e8bd2479212a3c2708f84692 100644 --- a/tests/app/v1/routers/test_entries_edit.py +++ b/tests/app/v1/routers/test_entries_edit.py @@ -16,22 +16,22 @@ # limitations under the License. # -import pytest from datetime import datetime -from nomad.search import search +import pytest + +from nomad import processing as proc +from nomad import utils from nomad.datamodel import Dataset -from nomad import processing as proc, utils +from nomad.search import search from nomad.utils.exampledata import ExampleData - from tests.app.v1.routers.common import assert_response from tests.processing.test_edit_metadata import ( - assert_metadata_edited, - all_coauthor_entry_metadata, all_admin_entry_metadata, + all_coauthor_entry_metadata, + assert_metadata_edited, ) - logger = utils.get_logger(__name__) diff --git a/tests/app/v1/routers/test_federation.py b/tests/app/v1/routers/test_federation.py index cb990557fa7cc14b526542074f65c298a93002cb..3b8d8284b51d58ccdd43b54bd5b1db150f616b88 100644 --- a/tests/app/v1/routers/test_federation.py +++ b/tests/app/v1/routers/test_federation.py @@ -1,9 +1,10 @@ -import requests import json -import pytest -from copy import deepcopy import logging import zlib +from copy import deepcopy + +import pytest +import requests from nomad.config import config from nomad.utils.structlogging import LogstashFormatter diff --git a/tests/app/v1/routers/test_graph.py b/tests/app/v1/routers/test_graph.py index 38f0670da8450ba2d01a751becf298292fd8ff79..5a12924f66a284f8d3af88036857ed051cf8b171 100644 --- a/tests/app/v1/routers/test_graph.py +++ b/tests/app/v1/routers/test_graph.py @@ -18,11 +18,11 @@ import pytest -from nomad.graph.graph_reader import EntryReader, Token from nomad.datamodel import EntryArchive +from nomad.datamodel.metainfo import SCHEMA_IMPORT_ERROR, simulationworkflowschema +from nomad.graph.graph_reader import EntryReader, Token from nomad.utils.exampledata import ExampleData from tests.archive.test_archive import assert_dict -from nomad.datamodel.metainfo import simulationworkflowschema, SCHEMA_IMPORT_ERROR # try: # from rich.pretty import pprint diff --git a/tests/app/v1/routers/test_materials.py b/tests/app/v1/routers/test_materials.py index 8490f3972414ba175a2859c228572d4a20aa92af..a1e998720b957664a3542fda24169942467dabae 100644 --- a/tests/app/v1/routers/test_materials.py +++ b/tests/app/v1/routers/test_materials.py @@ -16,27 +16,27 @@ # limitations under the License. # -import pytest from urllib.parse import urlencode -from nomad.metainfo.elasticsearch_extension import material_entry_type -from nomad.datamodel import results +import pytest +from nomad.datamodel import results +from nomad.metainfo.elasticsearch_extension import material_entry_type from tests.test_files import example_mainfile_contents # noqa: F401 from .common import ( aggregation_exclude_from_search_test_parameters, - assert_pagination, + aggregation_test_parameters_default, + assert_aggregation_response, assert_metadata_response, + assert_pagination, assert_required, + get_query_test_parameters, + owner_test_parameters, + pagination_test_parameters, perform_metadata_test, perform_owner_test, - owner_test_parameters, post_query_test_parameters, - get_query_test_parameters, - pagination_test_parameters, - aggregation_test_parameters_default, - assert_aggregation_response, ) """ diff --git a/tests/app/v1/routers/test_metainfo.py b/tests/app/v1/routers/test_metainfo.py index 8d4e2e78a44cde6573f411aac933f685601869d8..a34cb5be582d0e6cd8327987cfd4953f36ff45e2 100644 --- a/tests/app/v1/routers/test_metainfo.py +++ b/tests/app/v1/routers/test_metainfo.py @@ -21,11 +21,11 @@ from zipfile import ZipFile import pytest -from nomad.config import config from nomad.app.v1.routers.metainfo import store_package_definition -from nomad.datamodel import EntryArchive, ClientContext -from nomad.metainfo import MSection, MetainfoReferenceError -from nomad.utils import generate_entry_id, create_uuid +from nomad.config import config +from nomad.datamodel import ClientContext, EntryArchive +from nomad.metainfo import MetainfoReferenceError, MSection +from nomad.utils import create_uuid, generate_entry_id from tests.processing.test_data import run_processing diff --git a/tests/app/v1/routers/test_suggestions.py b/tests/app/v1/routers/test_suggestions.py index 937aad115af73b98413d9863cb7a857940bd35f2..093c20ed7eb23f288efbd91dfe87bb89e1cdd657 100644 --- a/tests/app/v1/routers/test_suggestions.py +++ b/tests/app/v1/routers/test_suggestions.py @@ -27,8 +27,10 @@ to assert for certain aspects in the responses. """ import pytest + from nomad.metainfo.elasticsearch_extension import entry_type from nomad.utils.exampledata import ExampleData + from .common import assert_response diff --git a/tests/app/v1/routers/test_systems.py b/tests/app/v1/routers/test_systems.py index debb8bdef085c2f6a3950386e9cbb53c7a161309..6b76eb777c317dfd6b6bf06550c4258914f08c3e 100644 --- a/tests/app/v1/routers/test_systems.py +++ b/tests/app/v1/routers/test_systems.py @@ -19,21 +19,21 @@ import re from io import BytesIO, StringIO -import pytest -import numpy as np import ase.io +import numpy as np +import pytest from ase import Atoms as ASEAtoms -from nomad.units import ureg -from nomad.normalizing.common import ase_atoms_from_nomad_atoms +from nomad.app.v1.routers.systems import FormatFeature, WrapModeEnum, format_map from nomad.datamodel.datamodel import EntryArchive -from nomad.datamodel.results import Results, Material, System from nomad.datamodel.metainfo import runschema from nomad.datamodel.metainfo.system import Atoms +from nomad.datamodel.results import Material, Results, System +from nomad.normalizing.common import ase_atoms_from_nomad_atoms +from nomad.units import ureg from nomad.utils.exampledata import ExampleData -from nomad.app.v1.routers.systems import format_map, FormatFeature, WrapModeEnum -from .common import assert_response, assert_browser_download_headers +from .common import assert_browser_download_headers, assert_response def ase_atoms(content, format): diff --git a/tests/app/v1/routers/uploads/test_basic_uploads.py b/tests/app/v1/routers/uploads/test_basic_uploads.py index 6546dd8b7de3dc9e640f652c67ef7a5cef99d2ca..a31f7ca2c20bdeadacf48f723536ef0954810e6f 100644 --- a/tests/app/v1/routers/uploads/test_basic_uploads.py +++ b/tests/app/v1/routers/uploads/test_basic_uploads.py @@ -18,12 +18,12 @@ import io import os +import tempfile import time import zipfile +from collections.abc import Iterable from datetime import datetime -import tempfile from typing import Any -from collections.abc import Iterable import pytest import requests @@ -41,15 +41,15 @@ from tests.app.v1.routers.common import ( assert_response, perform_get, ) +from tests.config.models.test_plugins import ( + mock_example_upload_entry_point, + mock_plugin_package, +) from tests.processing.test_edit_metadata import ( all_admin_metadata, all_coauthor_metadata, assert_metadata_edited, ) -from tests.config.models.test_plugins import ( - mock_plugin_package, - mock_example_upload_entry_point, -) from tests.test_files import ( assert_upload_files, empty_file, diff --git a/tests/app/v1/test_models.py b/tests/app/v1/test_models.py index 7328abd09346fd1dd42729f3ed36ddaa844e4396..9210f75d212792c627643cf23845572c02f11926 100644 --- a/tests/app/v1/test_models.py +++ b/tests/app/v1/test_models.py @@ -17,14 +17,16 @@ # from __future__ import annotations + +import sys + import pytest -from pydantic import BaseModel, Field, ValidationError import yaml -import sys +from pydantic import BaseModel, Field, ValidationError -from nomad.utils import strip from nomad.app.v1.models.graph import GraphRequest from nomad.app.v1.models.graph.utils import generate_request_model, mapped +from nomad.utils import strip @pytest.fixture() diff --git a/tests/app/v1/test_utils.py b/tests/app/v1/test_utils.py index ba6fd19fcfec71d55476496d73df32b2bd351f87..5ed2c55176c8fc573420fafce25fa53937d0f9d3 100644 --- a/tests/app/v1/test_utils.py +++ b/tests/app/v1/test_utils.py @@ -1,4 +1,5 @@ import pytest + from nomad.app.v1.utils import get_query_keys diff --git a/tests/archive/test_archive.py b/tests/archive/test_archive.py index 84e133d29ac6157ee8e34a80767e3a002e0f3ae3..0c35c9b5b0b8b7d3e89a829d6e5c1c1b2efd2768 100644 --- a/tests/archive/test_archive.py +++ b/tests/archive/test_archive.py @@ -15,42 +15,42 @@ # See the License for the specific language governing permissions and # limitations under the License. # +import json +import os.path from datetime import datetime -from typing import Any -import pytest -import msgpack from io import BytesIO -import os.path -import json +from typing import Any +import msgpack +import pytest import yaml from nomad import utils -from nomad.config import config +from nomad.archive import ( + ArchiveQueryError, + RequiredReader, + compute_required_with_referenced, + create_partial_archive, + query_archive, + read_archive, + read_partial_archive_from_mongo, + read_partial_archives_from_mongo, + write_archive, + write_partial_archive_to_mongo, +) from nomad.archive.converter import convert_archive +from nomad.archive.storage import _decode, _entries_per_block, to_json +from nomad.config import config +from nomad.datamodel import ClientContext, EntryArchive from nomad.metainfo import ( + Context, + MetainfoError, + MProxy, MSection, Quantity, + QuantityReference, Reference, SubSection, - QuantityReference, - MetainfoError, - Context, - MProxy, -) -from nomad.datamodel import EntryArchive, ClientContext -from nomad.archive.storage import _decode, _entries_per_block, to_json -from nomad.archive import ( - write_archive, - read_archive, - ArchiveQueryError, - query_archive, - write_partial_archive_to_mongo, - read_partial_archive_from_mongo, - read_partial_archives_from_mongo, - create_partial_archive, - compute_required_with_referenced, - RequiredReader, ) from nomad.utils.exampledata import ExampleData diff --git a/tests/archive/test_storage.py b/tests/archive/test_storage.py index 4a2de3dbf6bce3c2e3b9ab8bfd8553060b55f3f7..6ce614b53e82aac86d9a5570a4b2801352312cee 100644 --- a/tests/archive/test_storage.py +++ b/tests/archive/test_storage.py @@ -5,10 +5,7 @@ import msgpack import msgspec.msgpack import pytest -from nomad.archive.storage_v2 import ( - ArchiveList, - ArchiveDict, -) +from nomad.archive.storage_v2 import ArchiveDict, ArchiveList def generate_random_json(depth=10, width=4, simple=False): diff --git a/tests/config/models/test_plugins.py b/tests/config/models/test_plugins.py index a7d2f42ebb06ad971cd2aa00d806058927a4b68c..7d702f33e7f539100f52c98b90559b31bac11af3 100644 --- a/tests/config/models/test_plugins.py +++ b/tests/config/models/test_plugins.py @@ -18,12 +18,13 @@ import os import tempfile + import pytest -from nomad.config import Config +from nomad.config import Config from nomad.config.models.plugins import ( - ExampleUploadEntryPoint, APIEntryPoint, + ExampleUploadEntryPoint, UploadResource, ) diff --git a/tests/config/models/test_ui.py b/tests/config/models/test_ui.py index ce0626c7c885db5ad1184e7f0cddef0da262e6a5..606e9af0fb35890d54785ef54811806a8b562e34 100644 --- a/tests/config/models/test_ui.py +++ b/tests/config/models/test_ui.py @@ -22,13 +22,13 @@ from nomad.config.models.ui import ( App, Axis, AxisQuantity, - Columns, Column, - Rows, + Columns, RowActions, + RowActionURL, RowDetails, + Rows, RowSelection, - RowActionURL, ) diff --git a/tests/conftest.py b/tests/conftest.py index 66c1faa36e993a27d4cc28b2a6ae6296e83f52ef..a72d3e2687faca5ff45a9fbf00d1cf1e27a3da6a 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -16,17 +16,17 @@ # limitations under the License. # import builtins -from pathlib import Path -from io import StringIO -import pytest -import time -import os import logging -import warnings +import os +import socketserver import tempfile +import time +import warnings +from io import StringIO +from pathlib import Path +import pytest from fastapi.testclient import TestClient -import socketserver from nomad.config import config from nomad.config.models.plugins import Schema, add_plugin, remove_plugin @@ -35,8 +35,8 @@ from nomad.config.models.plugins import Schema, add_plugin, remove_plugin config.logstash.enabled = False # noqa: E402 # this must be set *before* the other modules are imported from nomad import utils -from nomad.utils import structlogging from nomad.app.main import app +from nomad.utils import structlogging # Set up pytest to pass control to the debugger on an exception. if os.getenv('_PYTEST_RAISE', '0') != '0': @@ -68,8 +68,8 @@ pytest_plugins = ( ) -from structlog.testing import LogCapture import structlog +from structlog.testing import LogCapture @pytest.fixture(scope='function') diff --git a/tests/data/schemas/nomadschemaexample/schema.py b/tests/data/schemas/nomadschemaexample/schema.py index 2bf501f0067643332f9dd9d0f98312a11d090b1e..2f076b6d5734b77ccdad44b834aa11142d8beba3 100644 --- a/tests/data/schemas/nomadschemaexample/schema.py +++ b/tests/data/schemas/nomadschemaexample/schema.py @@ -1,16 +1,17 @@ +import numpy as np + +from nomad.datamodel.data import EntryData +from nomad.datamodel.metainfo.annotations import ELNAnnotation, ELNComponentEnum from nomad.metainfo import ( - Quantity, - Package, - Section, - MEnum, Datetime, + MEnum, MSection, - SubSection, + Package, + Quantity, + Section, SectionProxy, + SubSection, ) -from nomad.datamodel.data import EntryData -from nomad.datamodel.metainfo.annotations import ELNAnnotation, ELNComponentEnum -import numpy as np m_package = Package() diff --git a/tests/datamodel/metainfo/test_annotations.py b/tests/datamodel/metainfo/test_annotations.py index 2d0f472f42072296571776a88c3d9d122e7e51b0..fb1208394b8348cc54a73facb400d8b366df3a71 100644 --- a/tests/datamodel/metainfo/test_annotations.py +++ b/tests/datamodel/metainfo/test_annotations.py @@ -19,13 +19,13 @@ import pytest from pydantic import ValidationError -from nomad.metainfo import Quantity from nomad.datamodel.metainfo.annotations import ( - PlotAnnotation, ELNAnnotation, + PlotAnnotation, PlotlyGraphObjectAnnotation, ) from nomad.datamodel.metainfo.plot import PlotlyError +from nomad.metainfo import Quantity @pytest.mark.parametrize( diff --git a/tests/datamodel/metainfo/test_plotly.py b/tests/datamodel/metainfo/test_plotly.py index 63d6daf2176827b663d4130701745b7d1af9c27c..038cffbd0e329d4233c8d64c0dcab992dbffbcff 100644 --- a/tests/datamodel/metainfo/test_plotly.py +++ b/tests/datamodel/metainfo/test_plotly.py @@ -1,4 +1,5 @@ import json + from tests.normalizing.conftest import run_processing diff --git a/tests/datamodel/metainfo/test_substance.py b/tests/datamodel/metainfo/test_substance.py index 78c63e39a54c7ba8257f95167652b4698bb1b8a3..6b30e54d32b7e67e459cbe769f27e29082fb3128 100644 --- a/tests/datamodel/metainfo/test_substance.py +++ b/tests/datamodel/metainfo/test_substance.py @@ -16,12 +16,13 @@ # limitations under the License. # +import json from collections.abc import Iterable + import pytest -import json -from tests.normalizing.conftest import run_processing from nomad.datamodel.metainfo.basesections import v1 as basesections +from tests.normalizing.conftest import run_processing class MockResponse: diff --git a/tests/datamodel/test_context.py b/tests/datamodel/test_context.py index 189753fec57af51ca1be6ab8d1fa6f588c5db9e5..f8250df5fbd4d35c3590721e94b72b204d4bb80d 100644 --- a/tests/datamodel/test_context.py +++ b/tests/datamodel/test_context.py @@ -16,20 +16,20 @@ # limitations under the License. # +import json import os +import re import pytest -import json -import re -from nomad import utils, files, processing -from nomad.metainfo.metainfo import MSection -from nomad.parsing.parser import ArchiveParser +from nomad import files, processing, utils from nomad.datamodel import Context -from nomad.datamodel.context import ServerContext, ClientContext, parse_path +from nomad.datamodel.context import ClientContext, ServerContext, parse_path from nomad.datamodel.datamodel import EntryArchive, EntryMetadata -from nomad.processing import Upload, Entry, ProcessStatus from nomad.datamodel.metainfo import runschema +from nomad.metainfo.metainfo import MSection +from nomad.parsing.parser import ArchiveParser +from nomad.processing import Entry, ProcessStatus, Upload @pytest.fixture(scope='module') diff --git a/tests/datamodel/test_datamodel.py b/tests/datamodel/test_datamodel.py index 306f2ad7c94a1992608a75410009b80c9e9ffa3c..e0b263d41e6b0ed67e9ba8fb4b1c38b8a1587c26 100644 --- a/tests/datamodel/test_datamodel.py +++ b/tests/datamodel/test_datamodel.py @@ -21,11 +21,12 @@ A generator for random test calculations. """ import random -from essential_generators import DocumentGenerator + import pytest +from essential_generators import DocumentGenerator +from nomad.datamodel.metainfo import SCHEMA_IMPORT_ERROR, runschema from nomad.parsing.parsers import parser_dict -from nomad.datamodel.metainfo import runschema, SCHEMA_IMPORT_ERROR number_of = 20 @@ -72,7 +73,7 @@ low_numbers_for_geometries = [1, 2, 2, 3, 3, 4, 4] @pytest.mark.skipif(runschema is None, reason=SCHEMA_IMPORT_ERROR) def test_common_metainfo(): from runschema.run import Run - from runschema.system import System, Atoms + from runschema.system import Atoms, System run = Run() system = run.m_create(System) @@ -83,8 +84,8 @@ def test_common_metainfo(): @pytest.mark.skipif(runschema is None, reason=SCHEMA_IMPORT_ERROR) def test_vasp_metainfo(): - from runschema.run import Run from electronicparsers.vasp.metainfo import vasp # noqa: F401 + from runschema.run import Run run = Run() assert 'vasp_src_date' in run.m_def.all_quantities diff --git a/tests/datamodel/test_hdf5.py b/tests/datamodel/test_hdf5.py index bbdb88e143b4ca15d350410e413cf282231e0417..427b20ccd2b33db0263e020afc0601fb431500f3 100644 --- a/tests/datamodel/test_hdf5.py +++ b/tests/datamodel/test_hdf5.py @@ -16,19 +16,18 @@ # limitations under the License. # -import pytest -import numpy as np import os + import h5py +import numpy as np +import pytest from nomad import files, processing -from nomad.datamodel import EntryData, EntryArchive, EntryMetadata +from nomad.datamodel import EntryArchive, EntryData, EntryMetadata from nomad.datamodel.context import ServerContext +from nomad.datamodel.hdf5 import HDF5Dataset, HDF5Reference from nomad.metainfo import Quantity -from nomad.datamodel.hdf5 import HDF5Reference, HDF5Dataset - - external_file = 'tests/data/datamodel/context.h5' diff --git a/tests/datamodel/test_metadata.py b/tests/datamodel/test_metadata.py index 09cc397ba545f6498c6ba1dea1a59487955a9e5d..0d6c40fd9fb6bd13f6e3890956dd397d042ac2b8 100644 --- a/tests/datamodel/test_metadata.py +++ b/tests/datamodel/test_metadata.py @@ -16,15 +16,16 @@ # limitations under the License. # -import pytest -import numpy as np from datetime import datetime + +import numpy as np +import pytest import pytz -from nomad.metainfo import Quantity, MSection, SubSection, Datetime, MEnum -from nomad.datamodel.datamodel import EntryMetadata, SearchableQuantity, EntryArchive -from nomad.metainfo.elasticsearch_extension import schema_separator from nomad.datamodel import EntryData +from nomad.datamodel.datamodel import EntryArchive, EntryMetadata, SearchableQuantity +from nomad.metainfo import Datetime, MEnum, MSection, Quantity, SubSection +from nomad.metainfo.elasticsearch_extension import schema_separator from tests.variables import python_schema_name diff --git a/tests/datamodel/test_schema.py b/tests/datamodel/test_schema.py index 17ccdea3ab84a0a3eb396270e7183376ecd0c421..c06f260122cab7664f40ab2adc3940c1425e60fa 100644 --- a/tests/datamodel/test_schema.py +++ b/tests/datamodel/test_schema.py @@ -17,21 +17,21 @@ # import os.path + import pytest -from nomad.metainfo import MetainfoError from nomad.datamodel.context import ServerContext +from nomad.datamodel.data import AuthorReference, Query, UserReference from nomad.datamodel.datamodel import EntryArchive, EntryMetadata -from nomad.datamodel.data import UserReference, AuthorReference, Query -from nomad.datamodel.metainfo.annotations import valid_eln_types, valid_eln_components +from nomad.datamodel.metainfo.annotations import valid_eln_components, valid_eln_types +from nomad.metainfo import MetainfoError from nomad.metainfo.data_type import Datatype from nomad.parsing.parser import ArchiveParser from nomad.processing.data import Upload from nomad.utils import get_logger, strip - +from tests.metainfo.test_yaml_schema import yaml_to_package from tests.normalizing.conftest import run_normalize from tests.test_files import create_test_upload_files -from tests.metainfo.test_yaml_schema import yaml_to_package def test_schema_processing(raw_files_function, no_warn): diff --git a/tests/examples/test_archive_query.py b/tests/examples/test_archive_query.py index 86c0eaf8446d3bc4f06c1137a6faa7416c51ff00..5cad68b2d008bcc4dabad27cddd706c87f2f1e40 100644 --- a/tests/examples/test_archive_query.py +++ b/tests/examples/test_archive_query.py @@ -16,14 +16,13 @@ # limitations under the License. # -import os.path import importlib +import os.path import sys from nomad.utils.exampledata import ExampleData - -from tests.parsing.test_parsing import run_singular_parser from tests.normalizing.conftest import run_normalize +from tests.parsing.test_parsing import run_singular_parser from tests.test_client import async_api_v1 # noqa: F401 diff --git a/tests/examples/test_docs.py b/tests/examples/test_docs.py index e8b737a9fbc0a21a85cd8e5d7893d0b0a5080697..ef1a43720d217ff262683120194593a606e4bd26 100644 --- a/tests/examples/test_docs.py +++ b/tests/examples/test_docs.py @@ -1,12 +1,12 @@ -import yaml import json import os.path +import yaml + from nomad.metainfo import Package from nomad.units import ureg - -from tests.parsing.test_parsing import run_singular_parser from tests.normalizing.conftest import run_normalize +from tests.parsing.test_parsing import run_singular_parser def _file(path): diff --git a/tests/examples/test_metainfo.py b/tests/examples/test_metainfo.py index 3c42051e03012def7b8162e687854b8f7910e246..739280867837e9d5cf4ef9ccf9d6fb1844f29468 100644 --- a/tests/examples/test_metainfo.py +++ b/tests/examples/test_metainfo.py @@ -1,7 +1,7 @@ -import runpy -import pytest import os +import runpy +import pytest prefix = os.path.join(__file__, '../../../examples/metainfo') diff --git a/tests/fixtures/data.py b/tests/fixtures/data.py index a041704c3547aff25501c360938984cd02f39064..d5000b0c381251ea99c04fd1fbf1908c08d5f788 100644 --- a/tests/fixtures/data.py +++ b/tests/fixtures/data.py @@ -17,7 +17,6 @@ from nomad.datamodel.datamodel import SearchableQuantity from nomad.metainfo.elasticsearch_extension import schema_separator from nomad.processing import ProcessStatus from nomad.utils.exampledata import ExampleData -from tests.variables import python_schema_name, yaml_schema_name, yaml_schema_root from tests.normalizing.conftest import run_normalize from tests.parsing import test_parsing from tests.processing import test_data as test_processing @@ -27,6 +26,7 @@ from tests.utils import ( create_template_upload_file, set_upload_entry_metadata, ) +from tests.variables import python_schema_name, yaml_schema_name, yaml_schema_root @pytest.fixture(scope='session') diff --git a/tests/fixtures/group_uploads.py b/tests/fixtures/group_uploads.py index b0d8109c89dfc6a64e179ae0e36233d505ff6331..290b2b6d0b44eb83eb8533be38b48528687f7d6d 100644 --- a/tests/fixtures/group_uploads.py +++ b/tests/fixtures/group_uploads.py @@ -11,6 +11,7 @@ Values: """ from collections.abc import Sequence + import pytest from nomad.utils.exampledata import ExampleData diff --git a/tests/graph/test_definition_reader.py b/tests/graph/test_definition_reader.py index f67385897f1c1994fd2602742007eb02aff75e4d..354f9fc00e38e17b3f63649fa45476bf35c942da 100644 --- a/tests/graph/test_definition_reader.py +++ b/tests/graph/test_definition_reader.py @@ -21,8 +21,8 @@ import pytest from nomad.graph.graph_reader import DefinitionReader from nomad.metainfo import ( - Package, MSection, + Package, Quantity, Reference, SectionProxy, diff --git a/tests/graph/test_graph_reader.py b/tests/graph/test_graph_reader.py index ebcdc887caf49afd4a80c6b9011198caed2622ba..d6f9d75cd2a38a1a978894c08620a68dad10b005 100644 --- a/tests/graph/test_graph_reader.py +++ b/tests/graph/test_graph_reader.py @@ -22,17 +22,17 @@ from datetime import datetime import pytest import yaml +from nomad.datamodel import EntryArchive from nomad.graph.graph_reader import ( EntryReader, - UploadReader, - UserReader, FileSystemReader, - MongoReader, GeneralReader, + MongoReader, Token, + UploadReader, + UserReader, ) from nomad.graph.lazy_wrapper import LazyWrapper -from nomad.datamodel import EntryArchive from nomad.utils.exampledata import ExampleData from tests.normalizing.conftest import simulationworkflowschema diff --git a/tests/metainfo/test_attributes.py b/tests/metainfo/test_attributes.py index 13edf2b9d098a417b6ad9508697a78415bcff20c..32db3438decb3c5868d99cf098d2d3bc88ea25ed 100644 --- a/tests/metainfo/test_attributes.py +++ b/tests/metainfo/test_attributes.py @@ -18,20 +18,20 @@ import datetime -import pytest import numpy as np +import pytest import pytz from nomad.metainfo import ( - MSection, - Quantity, Attribute, + Datetime, MEnum, + MSection, + Quantity, Reference, - Datetime, Section, ) -from nomad.metainfo.metainfo import MQuantity, Definition +from nomad.metainfo.metainfo import Definition, MQuantity from nomad.metainfo.util import validate_allowable_unit from nomad.units import ureg diff --git a/tests/metainfo/test_data_frames.py b/tests/metainfo/test_data_frames.py index b32aff08671e22390339e73d0185c3bbb333f609..9f1ee888517fef141fb712af31c1f3be6a728c50 100644 --- a/tests/metainfo/test_data_frames.py +++ b/tests/metainfo/test_data_frames.py @@ -2,22 +2,13 @@ import datetime import numpy as np import pandas as pd -import xarray as xr import pytest +import xarray as xr -from nomad.metainfo.metainfo import ( - MSection, - Package, - MEnum, - Datetime, -) -from nomad.metainfo.data_frames import ( - ValuesTemplate, - DataFrameTemplate, -) +from nomad.metainfo.data_frames import DataFrameTemplate, ValuesTemplate +from nomad.metainfo.metainfo import Datetime, MEnum, MSection, Package from nomad.units import ureg - m_package = Package() # Values diff --git a/tests/metainfo/test_elasticsearch_extension.py b/tests/metainfo/test_elasticsearch_extension.py index 6001560fb83995648c7700bef1f1b31b6c0e4583..e9aafb1575baa292c688786fd5d8429d380f1a99 100644 --- a/tests/metainfo/test_elasticsearch_extension.py +++ b/tests/metainfo/test_elasticsearch_extension.py @@ -17,28 +17,28 @@ # from datetime import date -import pytest + import numpy as np +import pytest from elasticsearch_dsl import Keyword from nomad.config import config -from nomad.utils.exampledata import ExampleData from nomad.datamodel.datamodel import SearchableQuantity -from nomad.metainfo import MSection, Quantity, SubSection, Datetime, Unit, MEnum +from nomad.metainfo import Datetime, MEnum, MSection, Quantity, SubSection, Unit from nomad.metainfo.elasticsearch_extension import ( Elasticsearch, create_indices, - index_entries_with_materials, + create_searchable_quantity, + entry_index, entry_type, - material_type, + index_entries_with_materials, material_entry_type, - entry_index, material_index, - create_searchable_quantity, + material_type, ) - -from tests.fixtures.infrastructure import clear_elastic_infra +from nomad.utils.exampledata import ExampleData from tests.app.v1.routers.common import perform_quantity_search_test +from tests.fixtures.infrastructure import clear_elastic_infra @pytest.fixture(scope='module') diff --git a/tests/metainfo/test_full_storage_quantity.py b/tests/metainfo/test_full_storage_quantity.py index 9acb42dcf198b1c1b404dda216cd464160a31381..e08520aad6f4c8e7d27574b015acba2424922218 100644 --- a/tests/metainfo/test_full_storage_quantity.py +++ b/tests/metainfo/test_full_storage_quantity.py @@ -18,12 +18,12 @@ import pytest from nomad.metainfo import ( - MSection, - Quantity, Attribute, - SubSection, MetainfoError, + MSection, + Quantity, Section, + SubSection, ) from nomad.metainfo.util import MQuantity from nomad.units import ureg diff --git a/tests/metainfo/test_hash_id.py b/tests/metainfo/test_hash_id.py index 3c3e44bb2e08569e30e48a0236f148d6b0109e2b..6e2b678431a1432dd43dce2d6475071a190e8e90 100644 --- a/tests/metainfo/test_hash_id.py +++ b/tests/metainfo/test_hash_id.py @@ -1,4 +1,4 @@ -from nomad.metainfo import Quantity, MSection, MEnum +from nomad.metainfo import MEnum, MSection, Quantity def simple_quantity(): diff --git a/tests/metainfo/test_metainfo.py b/tests/metainfo/test_metainfo.py index ea08bba6ca3b6af8324e339dd3db219f178e9207..394d44b588d6d341075f5eefde1857f2bfa15e26 100644 --- a/tests/metainfo/test_metainfo.py +++ b/tests/metainfo/test_metainfo.py @@ -20,40 +20,33 @@ # in-depth tests in test_* files of the same module. from math import isnan -import pytest + import numpy as np import pandas as pd import pint.quantity +import pytest -from nomad.metainfo.metainfo import ( - MSection, - MCategory, - Section, - Quantity, - SubSection, - Definition, - Package, - DeriveError, - MetainfoError, - derived, -) from nomad.metainfo import ( Annotation, + AnnotationModel, DefinitionAnnotation, SectionAnnotation, - AnnotationModel, ) -from nomad.metainfo.example import ( - Run, - VaspRun, - System, - SystemHash, - Parsing, - SCC, - m_package as example_package, +from nomad.metainfo.example import SCC, Parsing, Run, System, SystemHash, VaspRun +from nomad.metainfo.example import m_package as example_package +from nomad.metainfo.metainfo import ( + Definition, + DeriveError, + MCategory, + MetainfoError, + MSection, + Package, + Quantity, + Section, + SubSection, + derived, ) from nomad.units import ureg - from tests.metainfo import MTypes diff --git a/tests/metainfo/test_mongodb_extension.py b/tests/metainfo/test_mongodb_extension.py index 3f03c65a6dc97bb47f92e9532971d2e31d38b5f3..ff1687b7660e10954fc98b88fc9283ddd9160b32 100644 --- a/tests/metainfo/test_mongodb_extension.py +++ b/tests/metainfo/test_mongodb_extension.py @@ -17,9 +17,11 @@ # import json + import numpy as np -from nomad.metainfo import MSection, Section, Quantity, SubSection -from nomad.metainfo.mongoengine_extension import MongoDocument, Mongo + +from nomad.metainfo import MSection, Quantity, Section, SubSection +from nomad.metainfo.mongoengine_extension import Mongo, MongoDocument class B(MSection): diff --git a/tests/metainfo/test_package.py b/tests/metainfo/test_package.py index 75d70c3d2c77a30e94bb0c38c65f83585e52b350..583ccb35e6996ba10de5dc7825cb4eaa82795ed4 100644 --- a/tests/metainfo/test_package.py +++ b/tests/metainfo/test_package.py @@ -16,7 +16,7 @@ # limitations under the License. # -from nomad.metainfo import Package, MSection +from nomad.metainfo import MSection, Package m_package = Package(aliases=['nomad.datamodel.test_package']) diff --git a/tests/metainfo/test_quantities.py b/tests/metainfo/test_quantities.py index ad39b0e06313fce8e2dfffb4b6845d420cd79adb..0e6842b2d32fd941bbf2525a576b4d2c217c342f 100644 --- a/tests/metainfo/test_quantities.py +++ b/tests/metainfo/test_quantities.py @@ -24,14 +24,14 @@ import pytest import pytz from nomad.metainfo.metainfo import ( + JSON, + URL, Bytes, Capitalized, Datetime, Dimension, - JSON, MSection, Quantity, - URL, Unit, units, ) diff --git a/tests/metainfo/test_references.py b/tests/metainfo/test_references.py index c9f20fcc985f1ecdc725f3abbe401a6e92fc1e53..8a7f14a00a67c4cac18a049e50c84fa4b6ab52c2 100644 --- a/tests/metainfo/test_references.py +++ b/tests/metainfo/test_references.py @@ -16,25 +16,26 @@ # limitations under the License. # +import os.path from typing import cast + import pytest -import os.path -from nomad.datamodel import UserReference, AuthorReference +from nomad.datamodel import AuthorReference, UserReference from nomad.metainfo import ( + Context, + File, + MetainfoReferenceError, + MProxy, MSection, + Package, Quantity, + QuantityReference, + Reference, Section, SubSection, - MProxy, - Reference, - QuantityReference, - File, - MetainfoReferenceError, - Package as MetainfoPackage, - Context, - Package, ) +from nomad.metainfo import Package as MetainfoPackage class Referenced(MSection): @@ -383,7 +384,7 @@ def test_def_reference(): @pytest.mark.parametrize('mainfile', ['intra-entry', 'inter-entry']) def test_parse_with_references(mainfile): - from nomad.client import parse, normalize_all + from nomad.client import normalize_all, parse entry_archive = parse( os.path.join( diff --git a/tests/metainfo/test_sections.py b/tests/metainfo/test_sections.py index 04de671f982ab274ef5ac1f6c363b12a04cad2ab..7ed0276321249910bf0acb51232b3b50504d17b6 100644 --- a/tests/metainfo/test_sections.py +++ b/tests/metainfo/test_sections.py @@ -22,7 +22,7 @@ import pytest from nomad.metainfo import MSection -from nomad.metainfo.metainfo import Package, Quantity, SubSection, Section +from nomad.metainfo.metainfo import Package, Quantity, Section, SubSection def test_base_section(): @@ -251,7 +251,7 @@ def test_path(): assert SubSection.used_sections[ChildSection.m_def] == [EntryArchive.child] assert ChildSection.m_def.path == 'child' - from nomad.datamodel.metainfo.workflow import Workflow, Task + from nomad.datamodel.metainfo.workflow import Task, Workflow assert Workflow.m_def.path == 'workflow2' assert Task.m_def.path == '__no_archive_path__' diff --git a/tests/metainfo/test_to_dict.py b/tests/metainfo/test_to_dict.py index 91e2288fabdfb7fd847774e941682d114c5b9206..8c296a8868d559c7dea801d4bfebb38cfcb75dfa 100644 --- a/tests/metainfo/test_to_dict.py +++ b/tests/metainfo/test_to_dict.py @@ -16,16 +16,16 @@ # limitations under the License. # -import pytest import numpy as np +import pytest import yaml from nomad.app.v1.routers.metainfo import ( get_package_by_section_definition_id, store_package_definition, ) -from nomad.metainfo import MSection, MCategory, Quantity, SubSection -from nomad.metainfo.metainfo import Package, MEnum, Reference, Definition +from nomad.metainfo import MCategory, MSection, Quantity, SubSection +from nomad.metainfo.metainfo import Definition, MEnum, Package, Reference # resolve_references are tested in .test_references # type specific serialization is tested in .test_quantities diff --git a/tests/metainfo/test_yaml_schema.py b/tests/metainfo/test_yaml_schema.py index 67df9908508c25b5a6885ebe8279250379346687..950d8927c61b1ef84704caeb952c150034381599 100644 --- a/tests/metainfo/test_yaml_schema.py +++ b/tests/metainfo/test_yaml_schema.py @@ -20,18 +20,18 @@ import numpy as np import pytest import yaml -from nomad.utils import strip from nomad.metainfo import ( - Package, + Context, + MetainfoError, + MProxy, MSection, + Package, Quantity, Reference, - SubSection, Section, - MProxy, - MetainfoError, - Context, + SubSection, ) +from nomad.utils import strip m_package = Package() diff --git a/tests/mkdocs/test_mkdocs_metainfo.py b/tests/mkdocs/test_mkdocs_metainfo.py index 0ddda157dadb66ecc0281d78d5e470be3432f5a5..368288ff1cc8caf825772a45b86aeaf8981ef3d8 100644 --- a/tests/mkdocs/test_mkdocs_metainfo.py +++ b/tests/mkdocs/test_mkdocs_metainfo.py @@ -19,7 +19,7 @@ import pytest -from nomad.metainfo import MSection, Quantity, Datetime, Reference, Package +from nomad.metainfo import Datetime, MSection, Package, Quantity, Reference from nomad.mkdocs.metainfo import ( get_property_description, get_property_type_info, diff --git a/tests/mkdocs/test_mkdocs_pydantic.py b/tests/mkdocs/test_mkdocs_pydantic.py index 1183cde07a3666fd3c4c219e5bc0a71d17810255..15a9bda32ebfc639342c8f1f0ce92092d2a4736f 100644 --- a/tests/mkdocs/test_mkdocs_pydantic.py +++ b/tests/mkdocs/test_mkdocs_pydantic.py @@ -17,17 +17,18 @@ # from enum import Enum +from typing import Annotated, Literal, Optional, Union # type: ignore + import pytest -from typing import Union, Optional -from typing import Literal, Annotated # type: ignore from pydantic import BaseModel, Field + from nomad.config.models.ui import WidgetHistogram, WidgetTerms from nomad.mkdocs.pydantic import ( - get_field_type_info, - get_field_description, get_field_default, - get_field_options, get_field_deprecated, + get_field_description, + get_field_options, + get_field_type_info, ) diff --git a/tests/normalizing/conftest.py b/tests/normalizing/conftest.py index eac8d840eca71208d5b717f40e8b2e5114a99048..c9c926b861409de40f92b0f30362de8e13718788 100644 --- a/tests/normalizing/conftest.py +++ b/tests/normalizing/conftest.py @@ -15,53 +15,47 @@ # See the License for the specific language governing permissions and # limitations under the License. # +import re from collections import defaultdict -import numpy as np from typing import Any +from warnings import warn + +import ase.build +import numpy as np import pytest from ase import Atoms -import ase.build -import re -from warnings import warn -from nomad.units import ureg -from nomad.utils import get_logger -from nomad.normalizing import normalizers -from nomad.metainfo import SubSection, Quantity -from nomad.datamodel import EntryArchive, ArchiveSection -from nomad.datamodel.results import ( - Relation, - SymmetryNew as Symmetry, - Cell, - WyckoffSet, - System as ResultSystem, -) -from nomad.datamodel.optimade import Species -from nomad.normalizing.common import cell_from_ase_atoms, nomad_atoms_from_ase_atoms -from nomad.datamodel.metainfo.simulation.method import CoreHole -from nomad.datamodel.metainfo.workflow import Workflow +from nomad.datamodel import ArchiveSection, EntryArchive +from nomad.datamodel.context import ServerContext +from nomad.datamodel.datamodel import EntryArchive, EntryMetadata +from nomad.datamodel.metainfo import runschema, simulationworkflowschema from nomad.datamodel.metainfo.measurements import ( + EELSMeasurement, + Instrument, Measurement, Sample, - EELSMeasurement, Spectrum, - Instrument, ) -from nomad.datamodel.results import EELSInstrument - -from nomad.datamodel.context import ServerContext -from nomad.datamodel.datamodel import EntryArchive, EntryMetadata +from nomad.datamodel.metainfo.simulation.method import CoreHole +from nomad.datamodel.metainfo.workflow import Workflow +from nomad.datamodel.optimade import Species +from nomad.datamodel.results import Cell, EELSInstrument, Relation, WyckoffSet +from nomad.datamodel.results import SymmetryNew as Symmetry +from nomad.datamodel.results import System as ResultSystem +from nomad.metainfo import Quantity, SubSection +from nomad.normalizing import normalizers +from nomad.normalizing.common import cell_from_ase_atoms, nomad_atoms_from_ase_atoms from nomad.parsing.parser import ArchiveParser from nomad.processing.data import Upload -from tests.parsing.test_parsing import parsed_vasp_example # noqa: F401 -from tests.parsing.test_parsing import parsed_template_example # noqa: F401 -from tests.parsing.test_parsing import parsed_example # noqa: F401 -from tests.parsing.test_parsing import parse_file -from tests.test_files import create_test_upload_files -from nomad.datamodel.metainfo import ( - simulationworkflowschema, - runschema, +from nomad.units import ureg +from nomad.utils import get_logger +from tests.parsing.test_parsing import ( + parse_file, + parsed_example, # noqa: F401 + parsed_template_example, # noqa: F401 + parsed_vasp_example, # noqa: F401 ) +from tests.test_files import create_test_upload_files def run_normalize(entry_archive: EntryArchive) -> EntryArchive: diff --git a/tests/normalizing/test_entry_type_and_name.py b/tests/normalizing/test_entry_type_and_name.py index c0b4ba6a327a6defde96e39c33d7eb7f9cd34193..dffa84450dda6ebb4a292a09a2e706e4744ae2ef 100644 --- a/tests/normalizing/test_entry_type_and_name.py +++ b/tests/normalizing/test_entry_type_and_name.py @@ -16,7 +16,9 @@ # limitations under the License. # import pytest + import tests + from .conftest import run_normalize diff --git a/tests/normalizing/test_material.py b/tests/normalizing/test_material.py index c2b9289da2f68eb3c767fcfd0c2ea6bbf058dcb1..ab422266ef3aa1d335eae0318ad627944779e1bd 100644 --- a/tests/normalizing/test_material.py +++ b/tests/normalizing/test_material.py @@ -16,18 +16,18 @@ # limitations under the License. # +import ase.build import numpy as np import pytest from ase import Atoms -import ase.build from matid.symmetry.wyckoffset import WyckoffSet # pylint: disable=import-error -from nomad.units import ureg from nomad import atomutils from nomad.config import config -from nomad.utils import hash -from nomad.normalizing.common import ase_atoms_from_nomad_atoms from nomad.datamodel.results import ElementalComposition +from nomad.normalizing.common import ase_atoms_from_nomad_atoms +from nomad.units import ureg +from nomad.utils import hash from tests.normalizing.conftest import get_template_for_structure diff --git a/tests/normalizing/test_metainfo.py b/tests/normalizing/test_metainfo.py index 6fc097c0538354edfb6e2e99f2e87386acbfb943..815ad7df3a7882c324e4bf9b79655b3b0405bc93 100644 --- a/tests/normalizing/test_metainfo.py +++ b/tests/normalizing/test_metainfo.py @@ -17,9 +17,9 @@ # import numpy as np -from nomad.datamodel import EntryData, EntryArchive -from nomad.metainfo import Quantity, SubSection from nomad.client import normalize_all +from nomad.datamodel import EntryArchive, EntryData +from nomad.metainfo import Quantity, SubSection def test_normalizer_level(): diff --git a/tests/normalizing/test_method.py b/tests/normalizing/test_method.py index 483753d8e65c234b4ab841cadf39ad6c3632eed5..1f9992a15b47f9d19a6f8b11eb990aa0027e6866 100644 --- a/tests/normalizing/test_method.py +++ b/tests/normalizing/test_method.py @@ -17,9 +17,10 @@ # import numpy as np -from nomad.units import ureg import pytest +from nomad.units import ureg + def approx(value, abs=0, rel=1e-6): return pytest.approx(value, abs=abs, rel=rel) diff --git a/tests/normalizing/test_properties.py b/tests/normalizing/test_properties.py index ad82111b390280fb02528becf149458c473bcca5..9ad5ae24b2daf9179cab3d23ca33be0686e844fc 100644 --- a/tests/normalizing/test_properties.py +++ b/tests/normalizing/test_properties.py @@ -19,20 +19,20 @@ import numpy as np import pytest +from nomad.datamodel.metainfo import SCHEMA_IMPORT_ERROR, simulationworkflowschema from nomad.units import ureg from .conftest import ( - get_template_dft, - add_template_dos, - get_template_dos, add_template_band_structure, - get_template_band_structure, + add_template_dos, add_template_magnetic_shielding, - add_template_spin_spin_coupling, add_template_magnetic_susceptibility, + add_template_spin_spin_coupling, + get_template_band_structure, + get_template_dft, + get_template_dos, run_normalize, ) -from nomad.datamodel.metainfo import simulationworkflowschema, SCHEMA_IMPORT_ERROR def test_eels(eels): diff --git a/tests/normalizing/test_topology.py b/tests/normalizing/test_topology.py index 552949f1400c26e99927c556b8dae4382357c1e6..5940d8e8e9e8c1cbce921ca7adee123c90175eb2 100644 --- a/tests/normalizing/test_topology.py +++ b/tests/normalizing/test_topology.py @@ -16,34 +16,35 @@ # limitations under the License. # -import numpy as np from collections import defaultdict + +import numpy as np import pytest -from nomad.datamodel.metainfo import runschema +from nomad.datamodel.metainfo import runschema from nomad.units import ureg from tests.normalizing.conftest import ( # noqa: F401 - get_template_for_structure, - get_template_topology, + boron_nitride, + boron_nitride_topology, + check_template_active_orbitals, conv_bcc, conv_fcc, - rattle, - run_normalize, - stack, - surf, - single_cu_surface_topology, - single_cr_surface_topology, - stacked_cu_ni_surface_topology, + get_template_active_orbitals, + get_template_computation, + get_template_for_structure, + get_template_topology, graphene, graphene_topology, - boron_nitride, - boron_nitride_topology, mos2, mos2_topology, + rattle, + run_normalize, + single_cr_surface_topology, + single_cu_surface_topology, + stack, + stacked_cu_ni_surface_topology, stacked_graphene_boron_nitride_topology, - get_template_active_orbitals, - check_template_active_orbitals, - get_template_computation, + surf, ) diff --git a/tests/parsing/test_archive_parser.py b/tests/parsing/test_archive_parser.py index b9666aa564265de69aeb7d5bb64f9dcada4f8245..e577bbb0681973c580f493dcbc1e74ca96cc5782 100644 --- a/tests/parsing/test_archive_parser.py +++ b/tests/parsing/test_archive_parser.py @@ -16,14 +16,15 @@ # limitations under the License. # -import pytest import json import os import os.path +import pytest + from nomad.config import config +from nomad.datamodel import Context, EntryArchive from nomad.parsing.parser import ArchiveParser -from nomad.datamodel import EntryArchive, Context def test_archive_parser(raw_files_function): diff --git a/tests/parsing/test_file_parser.py b/tests/parsing/test_file_parser.py index b9ffdb65d35da55ddf0a24a7ee6469172d463012..d3a6e0a32c56d6c7f0443a8b54be2f3688dce1fa 100644 --- a/tests/parsing/test_file_parser.py +++ b/tests/parsing/test_file_parser.py @@ -1,14 +1,15 @@ -import pytest import numpy as np import pint +import pytest + +from nomad.datamodel.metainfo.system import Atoms from nomad.parsing.file_parser import ( - TextParser, - Quantity, + FileParser, ParsePattern, + Quantity, + TextParser, XMLParser, - FileParser, ) -from nomad.datamodel.metainfo.system import Atoms class TestFileParser: diff --git a/tests/parsing/test_mapping_parser.py b/tests/parsing/test_mapping_parser.py index 555587fab2626c22b5324989d82f5bee7c7b921f..a772a8b08cc1e8d9d00c5d9a99b281c1754d6563 100644 --- a/tests/parsing/test_mapping_parser.py +++ b/tests/parsing/test_mapping_parser.py @@ -1,28 +1,27 @@ -import pytest -import numpy as np -from typing import Any from copy import deepcopy +from typing import Any -from nomad.metainfo import Quantity, SubSection +import numpy as np +import pytest + +from nomad.datamodel import ArchiveSection from nomad.datamodel.metainfo.annotations import Mapper as MapperAnnotation +from nomad.metainfo import Quantity, SubSection from nomad.parsing.file_parser.mapping_parser import ( + MAPPING_ANNOTATION_KEY, + Data, + HDF5Parser, + Mapper, MappingParser, - XMLParser, MetainfoParser, - HDF5Parser, - TextParser, Path, - Transformer, - Data, - Mapper, PathParser, - MAPPING_ANNOTATION_KEY, -) -from nomad.parsing.file_parser.text_parser import ( - TextParser as TextFileParser, - Quantity as TextQuantity, + TextParser, + Transformer, + XMLParser, ) -from nomad.datamodel import ArchiveSection +from nomad.parsing.file_parser.text_parser import Quantity as TextQuantity +from nomad.parsing.file_parser.text_parser import TextParser as TextFileParser class BSection(ArchiveSection): diff --git a/tests/parsing/test_parsing.py b/tests/parsing/test_parsing.py index adfbece7378ccc90039d204d6b9a42ca1e502ba5..176679775b011990a7309df94b7dea15d2bc2e1f 100644 --- a/tests/parsing/test_parsing.py +++ b/tests/parsing/test_parsing.py @@ -19,13 +19,13 @@ import json import os from shutil import copyfile -from unittest.mock import patch, MagicMock +from unittest.mock import MagicMock, patch import pytest from nomad import files, utils from nomad.datamodel import EntryArchive -from nomad.parsing import BrokenParser, MatchingParserInterface, MatchingParser +from nomad.parsing import BrokenParser, MatchingParser, MatchingParserInterface from nomad.parsing.parsers import match_parser, parser_dict, parsers, run_parser from nomad.utils import dump_json diff --git a/tests/parsing/test_tabular.py b/tests/parsing/test_tabular.py index 50572e427ec98ce49c453fa9a94f75dfdf682317..7acf5017b8cbd3856ac4b7bb958a30f752878888 100644 --- a/tests/parsing/test_tabular.py +++ b/tests/parsing/test_tabular.py @@ -16,23 +16,23 @@ # limitations under the License. # -import pytest +import datetime import os import os.path import re -import datetime + +import pytest import yaml +from nomad import files from nomad.config import config -from nomad.datamodel.datamodel import EntryArchive, EntryMetadata from nomad.datamodel.context import ClientContext +from nomad.datamodel.datamodel import EntryArchive, EntryMetadata +from nomad.parsing.parser import ArchiveParser from nomad.parsing.tabular import read_table_data +from nomad.processing import Entry, ProcessStatus, Upload from nomad.utils import generate_entry_id, strip -from nomad.parsing.parser import ArchiveParser from tests.normalizing.conftest import run_normalize -from nomad.processing import Upload, Entry -from nomad.processing import ProcessStatus -from nomad import files def quantity_generator(quantity_name, header_name, shape, to_dict=False): diff --git a/tests/processing/test_base.py b/tests/processing/test_base.py index d1e2cf8ca3767bbfee3336a1135456f9e683e479..45fbe4560311a106976ca52e1cd65509564333f2 100644 --- a/tests/processing/test_base.py +++ b/tests/processing/test_base.py @@ -15,20 +15,20 @@ # See the License for the specific language governing permissions and # limitations under the License. # -import pytest import random -import time import threading +import time from typing import Any -from mongoengine import StringField, IntField, ListField +import pytest +from mongoengine import IntField, ListField, StringField from nomad.processing.base import ( Proc, ProcessAlreadyRunning, + ProcessStatus, process, process_local, - ProcessStatus, ) random.seed(0) diff --git a/tests/processing/test_data.py b/tests/processing/test_data.py index f32e0f14690757dfc44239645b754759ac376542..d538f39a91a3f52cfd8128a1bbbeda7da4edf1fd 100644 --- a/tests/processing/test_data.py +++ b/tests/processing/test_data.py @@ -16,39 +16,39 @@ # limitations under the License. # -from collections.abc import Generator -import pytest +import json import os.path import re import shutil import zipfile -import json +from collections.abc import Generator + +import pytest import yaml -from nomad import utils, infrastructure +from nomad import infrastructure, utils +from nomad.archive import read_partial_archive_from_mongo, to_json from nomad.config import config from nomad.config.models.config import BundleImportSettings -from nomad.archive import read_partial_archive_from_mongo, to_json -from nomad.files import UploadFiles, StagingUploadFiles, PublicUploadFiles -from nomad.parsing.parser import Parser -from nomad.parsing import parsers from nomad.datamodel import ServerContext from nomad.datamodel.data import EntryData +from nomad.datamodel.datamodel import ArchiveSection, EntryArchive, EntryData +from nomad.files import PublicUploadFiles, StagingUploadFiles, UploadFiles from nomad.metainfo import Package, Quantity, Reference, SubSection -from nomad.processing import Upload, Entry, ProcessStatus -from nomad.search import search, refresh as search_refresh +from nomad.parsing import parsers +from nomad.parsing.parser import Parser +from nomad.processing import Entry, ProcessStatus, Upload +from nomad.search import refresh as search_refresh +from nomad.search import search from nomad.utils.exampledata import ExampleData -from nomad.datamodel.datamodel import EntryArchive, EntryData, ArchiveSection - -from tests.test_search import assert_search_upload from tests.test_files import ( assert_upload_files, - example_file_mainfile, example_file_aux, + example_file_mainfile, ) +from tests.test_search import assert_search_upload from tests.utils import create_template_upload_file, set_upload_entry_metadata - # Package with some metainfo schemas used only for testing. m_package = Package(name='test_schemas') diff --git a/tests/processing/test_edit_metadata.py b/tests/processing/test_edit_metadata.py index 6d53da2b890cc7743bfbb0f527e40ff97713dbc0..96963944b85296fcf2e4c2418964f14ca579a514 100644 --- a/tests/processing/test_edit_metadata.py +++ b/tests/processing/test_edit_metadata.py @@ -15,18 +15,17 @@ # See the License for the specific language governing permissions and # limitations under the License. # -import pytest from datetime import datetime +import pytest from fastapi.exceptions import RequestValidationError from nomad import datamodel, metainfo from nomad.metainfo.data_type import Datatype -from nomad.processing import Upload, MetadataEditRequestHandler +from nomad.processing import MetadataEditRequestHandler, Upload from nomad.processing.data import editable_metadata, mongo_upload_metadata from nomad.search import search - all_coauthor_metadata = dict( # All attributes which a coauthor+ can edit upload_name='a humble upload name', diff --git a/tests/processing/test_rfc3161.py b/tests/processing/test_rfc3161.py index e559a53246882e5154175d2a570a6bc61bd8ca48..28f1bcc80ba94bbadbdc0bd0192b2e84786cd96a 100644 --- a/tests/processing/test_rfc3161.py +++ b/tests/processing/test_rfc3161.py @@ -23,9 +23,9 @@ import httpx import pytest import rfc3161ng -from nomad.archive import write_archive, read_archive, to_json +from nomad.archive import read_archive, to_json, write_archive from nomad.datamodel.datamodel import RFC3161Timestamp -from nomad.processing.data import get_rfc3161_token, Entry +from nomad.processing.data import Entry, get_rfc3161_token @pytest.mark.parametrize( diff --git a/tests/states/archives/create_archives.py b/tests/states/archives/create_archives.py index bcee46b0a307349c098720aed9eb54741e85b34a..0b53682f6dbcf3b0d580c005c09522d197edcf6c 100644 --- a/tests/states/archives/create_archives.py +++ b/tests/states/archives/create_archives.py @@ -17,6 +17,7 @@ # import math + from nomad.utils.exampledata import create_entry_archive diff --git a/tests/states/entry.py b/tests/states/entry.py index 767a77b37087849e30d2dfa81b850bf6edb45490..bbd97182e0c9389beb0731d255391002118eb053 100644 --- a/tests/states/entry.py +++ b/tests/states/entry.py @@ -17,10 +17,12 @@ # import json -from nomad import infrastructure, files + +from nomad import files, infrastructure +from nomad.processing import Upload from nomad.utils.exampledata import ExampleData, create_entry_archive + from .archives.create_archives import archive_dft_bulk -from nomad.processing import Upload def dft(): diff --git a/tests/states/search.py b/tests/states/search.py index e3ef1fc19c585b0ceb5f4cea7d443f897b0047af..6a53a211cc259d5579aa653b461a6c9ab2ff5864 100644 --- a/tests/states/search.py +++ b/tests/states/search.py @@ -16,10 +16,11 @@ # limitations under the License. # from datetime import datetime, timedelta + from nomad import infrastructure -from nomad.utils import create_uuid -from nomad.units import ureg from nomad.atomutils import chemical_symbols +from nomad.units import ureg +from nomad.utils import create_uuid from nomad.utils.exampledata import ExampleData material_h2o = { diff --git a/tests/states/uploads.py b/tests/states/uploads.py index 52ae1ed4e99129aef629545183df7d84693ee127..a79aa893b8bcb3aac07d135887748803d01aca5b 100644 --- a/tests/states/uploads.py +++ b/tests/states/uploads.py @@ -16,11 +16,12 @@ # limitations under the License. # -from nomad import infrastructure, files +from nomad import files, infrastructure from nomad.processing import Upload from nomad.utils.exampledata import ExampleData + from .archives.create_archives import archive_dft_bulk -from .groups import init_gui_test_groups, delete_group +from .groups import delete_group, init_gui_test_groups default_access = {'coauthors': ['scooper'], 'reviewers': ['ttester']} twin_access = { diff --git a/tests/test_atomutils.py b/tests/test_atomutils.py index 97655c0eded200dafd7bea8bf237a0bf35a3dd85..e9dc5f5561a5d24803bcc0369b5a081cc73b5ecc 100644 --- a/tests/test_atomutils.py +++ b/tests/test_atomutils.py @@ -16,8 +16,9 @@ # limitations under the License. # import pytest + from nomad.atomutils import Formula -from nomad.datamodel.results import Material, ElementalComposition +from nomad.datamodel.results import ElementalComposition, Material @pytest.mark.parametrize( diff --git a/tests/test_cli.py b/tests/test_cli.py index 3ee028698b5e5098b2761c0f25f3066d413e869e..1350894136449b375906f4937adf8020ef4cfaf0 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -16,20 +16,22 @@ # limitations under the License. # -import pytest -import click.testing +import datetime import json import os -import datetime -import time import tempfile +import time -from nomad import processing as proc, files -from nomad.config import config -from nomad.search import search +import click.testing +import pytest + +from nomad import files +from nomad import processing as proc from nomad.cli import cli from nomad.cli.cli import POPO -from nomad.processing import Upload, Entry, ProcessStatus +from nomad.config import config +from nomad.processing import Entry, ProcessStatus, Upload +from nomad.search import search from nomad.utils.exampledata import ExampleData # TODO there is much more to test diff --git a/tests/test_client.py b/tests/test_client.py index ce2f246cf51093841b4501c02b84b1874c05897f..35211bf701cd3aeed9391f3b6dc89707a19b5e68 100644 --- a/tests/test_client.py +++ b/tests/test_client.py @@ -15,28 +15,23 @@ # See the License for the specific language governing permissions and # limitations under the License. # +import json import os -from httpx import AsyncClient import pytest -import json - +from httpx import AsyncClient from pydantic import ValidationError from nomad.app.main import app from nomad.client.archive import ArchiveQuery from nomad.datamodel import EntryArchive, User -from nomad.datamodel.metainfo import runschema, SCHEMA_IMPORT_ERROR -from nomad.datamodel.metainfo.annotations import ( - Rule, - Rules, -) +from nomad.datamodel.metainfo import SCHEMA_IMPORT_ERROR, runschema +from nomad.datamodel.metainfo.annotations import Rule, Rules from nomad.metainfo import MSection, SubSection from nomad.utils.json_transformer import Transformer from tests.fixtures.users import users from tests.processing import test_data as test_processing - # TODO: more tests diff --git a/tests/test_common.py b/tests/test_common.py index 919538e69ef708a3e7d1fcec9b9ac4a090f83316..432be865b7eb05c5376cb65f6fda6915865fbda0 100644 --- a/tests/test_common.py +++ b/tests/test_common.py @@ -17,7 +17,8 @@ # import pytest -from nomad.common import is_safe_relative_path, is_safe_path + +from nomad.common import is_safe_path, is_safe_relative_path @pytest.mark.parametrize( diff --git a/tests/test_config.py b/tests/test_config.py index 7cc0f5070576d2c59b4fed4bd1acf8cebadb24e2..20fbe5d18f437c9b80ab850331410861264a4da8 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -18,12 +18,13 @@ import os import re + import pytest import yaml from pydantic import ValidationError from nomad.config import load_config -from nomad.config.models.plugins import Parser, Schema, ParserEntryPoint +from nomad.config.models.plugins import Parser, ParserEntryPoint, Schema from nomad.utils import flatten_dict from .utils import assert_log diff --git a/tests/test_doi.py b/tests/test_doi.py index 60498040f7d707584a64bc152bcaf5729b268f54..bcb4e61df01b8f202029b884856f0fb0976f3f22 100644 --- a/tests/test_doi.py +++ b/tests/test_doi.py @@ -15,10 +15,12 @@ # See the License for the specific language governing permissions and # limitations under the License. # +from unittest.mock import MagicMock + +import pytest + from nomad.config import config from nomad.doi import DOI, DOIException -import pytest -from unittest.mock import MagicMock def test_create(mongo_function, user1, no_warn): diff --git a/tests/test_files.py b/tests/test_files.py index 519be1ec049b27579b1d9c2ac7ae1eb6c482678b..e2ca1d432ffd9f22b104eb4356b5d3b2a0d6a01b 100644 --- a/tests/test_files.py +++ b/tests/test_files.py @@ -16,31 +16,33 @@ # limitations under the License. # -from typing import Any -from collections.abc import Generator, Iterable -from datetime import datetime +import itertools import os import os.path +import pathlib +import re import shutil -import pytest -import itertools import zipfile -import re -import pathlib +from collections.abc import Generator, Iterable +from datetime import datetime +from typing import Any + +import pytest from nomad import datamodel, utils -from nomad.config import config from nomad.archive import to_json +from nomad.config import config from nomad.files import ( DirectoryObject, PathObject, - empty_zip_file_size, + PublicUploadFiles, + StagingUploadFiles, + UploadFiles, empty_archive_file_size, + empty_zip_file_size, ) -from nomad.files import StagingUploadFiles, PublicUploadFiles, UploadFiles from nomad.processing import Upload - EntryWithFiles = tuple[datamodel.EntryMetadata, str] UploadWithFiles = tuple[str, list[datamodel.EntryMetadata], UploadFiles] StagingUploadWithFiles = tuple[str, list[datamodel.EntryMetadata], StagingUploadFiles] diff --git a/tests/test_logtransfer.py b/tests/test_logtransfer.py index b67262fbeedf50b7db7a447f8e0e58da2374c5fb..122c727b07a823fff4cd0d3293b405c4f687c394 100644 --- a/tests/test_logtransfer.py +++ b/tests/test_logtransfer.py @@ -1,11 +1,12 @@ -import logging import json -import pytest +import logging import os.path +import pytest + from nomad import config, utils -from nomad.utils import structlogging from nomad.logtransfer import transfer_logs +from nomad.utils import structlogging @pytest.fixture(scope='function') diff --git a/tests/test_search.py b/tests/test_search.py index 319dcbac437aa0c5ffe236838613beb43cea1598..0e6eeb1f709b52d9fff347b04f1f0a243ffe8edf 100644 --- a/tests/test_search.py +++ b/tests/test_search.py @@ -17,9 +17,9 @@ # import json +from collections.abc import Iterable from datetime import datetime from typing import Any -from collections.abc import Iterable import pytest @@ -41,15 +41,8 @@ from nomad.metainfo.elasticsearch_extension import ( ) from nomad.metainfo.metainfo import Datetime, Quantity from nomad.metainfo.util import MEnum -from nomad.search import ( - AuthenticationRequiredError as ARE, -) -from nomad.search import ( - quantity_values, - refresh, - search, - update_by_query, -) +from nomad.search import AuthenticationRequiredError as ARE +from nomad.search import quantity_values, refresh, search, update_by_query from nomad.utils import deep_get from nomad.utils.exampledata import ExampleData from tests.variables import python_schema_name, yaml_schema_name diff --git a/tests/test_test.py b/tests/test_test.py index bbca3fb46962648c5bad3662940c5f5101f761d5..ef2353b7ac48692678fcf6afcc0de6017e366d3d 100644 --- a/tests/test_test.py +++ b/tests/test_test.py @@ -16,9 +16,10 @@ # limitations under the License. # -import pytest import logging +import pytest + @pytest.fixture() def my_caplog(caplog): diff --git a/tests/test_utils.py b/tests/test_utils.py index 5d7339d68ef2b5de9c272aee1def2468d504d7ed..11f38f65f18387fd3e9db8633888d4009e239b69 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -16,24 +16,24 @@ # limitations under the License. # -from importlib.metadata import PackageNotFoundError import time -import pytest +from importlib.metadata import PackageNotFoundError + import pandas as pd +import pytest -from nomad import utils +from nomad import files, utils from nomad.metainfo.metainfo import MSection, Quantity, SubSection -from nomad import files from nomad.processing import Upload from nomad.utils import ( - nomad_distro_metadata, - structlogging, - flatten_dict, - rebuild_dict, - prune_dict, + dataframe_to_dict, deep_get, dict_to_dataframe, - dataframe_to_dict, + flatten_dict, + nomad_distro_metadata, + prune_dict, + rebuild_dict, + structlogging, ) diff --git a/tests/utils.py b/tests/utils.py index 253200de91c69f246f6f7245885fd250315dca8a..c24fe475952acda078d98d74bebfc980be751012 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -23,6 +23,7 @@ import urllib.parse import zipfile from logging import LogRecord from typing import Any + import pytest