Merge branch 'main' of https://gitlabext.wsl.ch/EnviDat/opendataswiss into main

.gitignore

 **/*.secret
-__pypackages__
+.pdm.toml
+pdm.lock
+__pypackages__/

README.md

+# OpenDataSwiss
+
+Code to make EnviDat data accessible via OpenDataSwiss scrapers.
+
+- Builds a Python image with all necessary requirements.
+- Adds the main.py file to the image.
+- Executes main.py at image startup.
+
+## Development
+
+- Build the debug image:
+  `docker compose build`
+
+- Create .app.secret:
+
+  ```env
+  LOG_LEVEL=DEBUG
+  AWS_ENDPOINT=xxx
+  AWS_REGION=xxx
+  AWS_ACCESS_KEY=xxx
+  AWS_SECRET_KEY=xxx
+  AWS_BUCKET_NAME=xxx
+  BACKEND_PACKAGE_URL=xxx
+  ```
+
+- Run via VSCode debug menu.
+
+## Production
+
+- Create the required secrets in the `cron` namespace.
+- Push the latest code.
+- Watch the build pipeline run in Gitlab.
+- Cronjob from the `k8s-cron` repo runs on schedule, using built image.

pdm.lock

-[[package]]
-name = "boto3"
-version = "1.21.23"
-requires_python = ">= 3.6"
-summary = "The AWS SDK for Python"
-dependencies = [
-    "botocore<1.25.0,>=1.24.23",
-    "jmespath<2.0.0,>=0.7.1",
-    "s3transfer<0.6.0,>=0.5.0",
-]
-
-[[package]]
-name = "botocore"
-version = "1.24.46"
-requires_python = ">= 3.6"
-summary = "Low-level, data-driven core of boto 3."
-dependencies = [
-    "jmespath<2.0.0,>=0.7.1",
-    "python-dateutil<3.0.0,>=2.1",
-    "urllib3<1.27,>=1.25.4",
-]
-
-[[package]]
-name = "certifi"
-version = "2021.10.8"
-summary = "Python package for providing Mozilla's CA Bundle."
-
-[[package]]
-name = "charset-normalizer"
-version = "2.0.12"
-requires_python = ">=3.5.0"
-summary = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet."
-
-[[package]]
-name = "idna"
-version = "3.3"
-requires_python = ">=3.5"
-summary = "Internationalized Domain Names in Applications (IDNA)"
-
-[[package]]
-name = "jmespath"
-version = "1.0.0"
-requires_python = ">=3.7"
-summary = "JSON Matching Expressions"
-
-[[package]]
-name = "python-dateutil"
-version = "2.8.2"
-requires_python = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7"
-summary = "Extensions to the standard Python datetime module"
-dependencies = [
-    "six>=1.5",
-]
-
-[[package]]
-name = "requests"
-version = "2.27.1"
-requires_python = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*"
-summary = "Python HTTP for Humans."
-dependencies = [
-    "certifi>=2017.4.17",
-    "charset-normalizer~=2.0.0; python_version >= \"3\"",
-    "idna<4,>=2.5; python_version >= \"3\"",
-    "urllib3<1.27,>=1.21.1",
-]
-
-[[package]]
-name = "s3transfer"
-version = "0.5.2"
-requires_python = ">= 3.6"
-summary = "An Amazon S3 Transfer Manager"
-dependencies = [
-    "botocore<2.0a.0,>=1.12.36",
-]
-
-[[package]]
-name = "six"
-version = "1.16.0"
-requires_python = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*"
-summary = "Python 2 and 3 compatibility utilities"
-
-[[package]]
-name = "urllib3"
-version = "1.26.9"
-requires_python = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, <4"
-summary = "HTTP library with thread-safe connection pooling, file post, and more."
-
-[metadata]
-lock_version = "3.1"
-content_hash = "sha256:2cb85462fdf5d1e9853719d5dde86216f1809350cf43429c5b3659aa2b22f554"
-
-[metadata.files]
-"boto3 1.21.23" = [
-    {file = "boto3-1.21.23-py3-none-any.whl", hash = "sha256:c60d2401226c723db809df5c611a850c101b562e0e0790be7631a38b82813038"},
-    {file = "boto3-1.21.23.tar.gz", hash = "sha256:44a30268702c78aa355a0044fa7fea5f1e7e7c125ad029196028787eea003c2e"},
-]
-"botocore 1.24.46" = [
-    {file = "botocore-1.24.46-py3-none-any.whl", hash = "sha256:663d8f02b98641846eb959c54c840cc33264d5f2dee5b8fc09ee8adbef0f8dcf"},
-    {file = "botocore-1.24.46.tar.gz", hash = "sha256:89a203bba3c8f2299287e48a9e112e2dbe478cf67eaac26716f0e7f176446146"},
-]
-"certifi 2021.10.8" = [
-    {file = "certifi-2021.10.8-py2.py3-none-any.whl", hash = "sha256:d62a0163eb4c2344ac042ab2bdf75399a71a2d8c7d47eac2e2ee91b9d6339569"},
-    {file = "certifi-2021.10.8.tar.gz", hash = "sha256:78884e7c1d4b00ce3cea67b44566851c4343c120abd683433ce934a68ea58872"},
-]
-"charset-normalizer 2.0.12" = [
-    {file = "charset_normalizer-2.0.12-py3-none-any.whl", hash = "sha256:6881edbebdb17b39b4eaaa821b438bf6eddffb4468cf344f09f89def34a8b1df"},
-    {file = "charset-normalizer-2.0.12.tar.gz", hash = "sha256:2857e29ff0d34db842cd7ca3230549d1a697f96ee6d3fb071cfa6c7393832597"},
-]
-"idna 3.3" = [
-    {file = "idna-3.3-py3-none-any.whl", hash = "sha256:84d9dd047ffa80596e0f246e2eab0b391788b0503584e8945f2368256d2735ff"},
-    {file = "idna-3.3.tar.gz", hash = "sha256:9d643ff0a55b762d5cdb124b8eaa99c66322e2157b69160bc32796e824360e6d"},
-]
-"jmespath 1.0.0" = [
-    {file = "jmespath-1.0.0-py3-none-any.whl", hash = "sha256:e8dcd576ed616f14ec02eed0005c85973b5890083313860136657e24784e4c04"},
-    {file = "jmespath-1.0.0.tar.gz", hash = "sha256:a490e280edd1f57d6de88636992d05b71e97d69a26a19f058ecf7d304474bf5e"},
-]
-"python-dateutil 2.8.2" = [
-    {file = "python_dateutil-2.8.2-py2.py3-none-any.whl", hash = "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9"},
-    {file = "python-dateutil-2.8.2.tar.gz", hash = "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86"},
-]
-"requests 2.27.1" = [
-    {file = "requests-2.27.1-py2.py3-none-any.whl", hash = "sha256:f22fa1e554c9ddfd16e6e41ac79759e17be9e492b3587efa038054674760e72d"},
-    {file = "requests-2.27.1.tar.gz", hash = "sha256:68d7c56fd5a8999887728ef304a6d12edc7be74f1cfa47714fc8b414525c9a61"},
-]
-"s3transfer 0.5.2" = [
-    {file = "s3transfer-0.5.2-py3-none-any.whl", hash = "sha256:7a6f4c4d1fdb9a2b640244008e142cbc2cd3ae34b386584ef044dd0f27101971"},
-    {file = "s3transfer-0.5.2.tar.gz", hash = "sha256:95c58c194ce657a5f4fb0b9e60a84968c808888aed628cd98ab8771fe1db98ed"},
-]
-"six 1.16.0" = [
-    {file = "six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"},
-    {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"},
-]
-"urllib3 1.26.9" = [
-    {file = "urllib3-1.26.9-py2.py3-none-any.whl", hash = "sha256:44ece4d53fb1706f667c9bd1c648f5469a2ec925fcf3a776667042d645472c14"},
-    {file = "urllib3-1.26.9.tar.gz", hash = "sha256:aabaf16477806a5e1dd19aa41f8c2b7950dd3c746362d7e3223dbe6de6ac448e"},
-]

pyproject.toml

@@ -4,9 +4,12 @@ version = "0.1.0"
 description = ""
 authors = [
     {name = "Sam Woodcock", email = "samuel.woodcock@wsl.ch"},
-    {name= "Rebecca Kurup Buchholz"}
+    {name= "Rebecca Kurup Buchholz", email = "None"}
+]
+dependencies = [
+    "s3-bucket>=1.4.0",
+    "requests==2.27.1",
 ]
-dependencies = ["boto3==1.21.23", "requests==2.27.1"]
 requires-python = ">=3.9"
 license = {text = "MIT"}