diff --git a/spark/s3_tools.py b/app/common/s3.py similarity index 95% rename from spark/s3_tools.py rename to app/common/s3.py index 4eebb671..329ee75c 100644 --- a/spark/s3_tools.py +++ b/app/common/s3.py @@ -2,10 +2,10 @@ import boto3 from botocore.client import Config -from spark import s3_buckets, s3_clusters, get_cluster_name, s3_users +from app.common import s3_buckets, s3_clusters, get_cluster_name, s3_users import re import random -from typing import Dict, Iterator, List, Tuple, Union +from typing import List, Union __re_s3_path = re.compile("^s3a?://([^/]+)(?:/(.*))?$") def get_s3_config(path: Union[str, List[str]], outside=False): diff --git a/test/__init__.py b/pipeline/__init__.py similarity index 100% rename from test/__init__.py rename to pipeline/__init__.py diff --git a/test/test_commons.py b/test/test_commons.py index 823f3bfa..4af5ef9e 100644 --- a/test/test_commons.py +++ b/test/test_commons.py @@ -1,10 +1,9 @@ import io import json import os -import unittest from libs.commons import fitz -from spark.s3_tools import get_s3_config, get_s3_client +from app.common.s3 import get_s3_config, get_s3_client from libs.commons import join_path, json_dump_path, read_file, parse_bucket_key from loguru import logger diff --git a/test/test_para/__init__.py b/test/test_para/__init__.py deleted file mode 100644 index e69de29b..00000000