WSL/SLF GitLab Repository

Commit f09ced4e authored by Rebecca Kurup Buchholz's avatar Rebecca Kurup Buchholz
Browse files

Merge branch 'main' of https://gitlabext.wsl.ch/EnviDat/opendataswiss into main

parents e4a01630 bccda034
**/*.secret
__pypackages__
.pdm.toml
pdm.lock
__pypackages__/
# OpenDataSwiss
Code to make EnviDat data accessible via OpenDataSwiss scrapers.
- Builds a Python image with all necessary requirements.
- Adds the main.py file to the image.
- Executes main.py at image startup.
## Development
- Build the debug image:
`docker compose build`
- Create .app.secret:
```env
LOG_LEVEL=DEBUG
AWS_ENDPOINT=xxx
AWS_REGION=xxx
AWS_ACCESS_KEY=xxx
AWS_SECRET_KEY=xxx
AWS_BUCKET_NAME=xxx
BACKEND_PACKAGE_URL=xxx
```
- Run via VSCode debug menu.
## Production
- Create the required secrets in the `cron` namespace.
- Push the latest code.
- Watch the build pipeline run in Gitlab.
- Cronjob from the `k8s-cron` repo runs on schedule, using built image.
[[package]]
name = "boto3"
version = "1.21.23"
requires_python = ">= 3.6"
summary = "The AWS SDK for Python"
dependencies = [
"botocore<1.25.0,>=1.24.23",
"jmespath<2.0.0,>=0.7.1",
"s3transfer<0.6.0,>=0.5.0",
]
[[package]]
name = "botocore"
version = "1.24.46"
requires_python = ">= 3.6"
summary = "Low-level, data-driven core of boto 3."
dependencies = [
"jmespath<2.0.0,>=0.7.1",
"python-dateutil<3.0.0,>=2.1",
"urllib3<1.27,>=1.25.4",
]
[[package]]
name = "certifi"
version = "2021.10.8"
summary = "Python package for providing Mozilla's CA Bundle."
[[package]]
name = "charset-normalizer"
version = "2.0.12"
requires_python = ">=3.5.0"
summary = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet."
[[package]]
name = "idna"
version = "3.3"
requires_python = ">=3.5"
summary = "Internationalized Domain Names in Applications (IDNA)"
[[package]]
name = "jmespath"
version = "1.0.0"
requires_python = ">=3.7"
summary = "JSON Matching Expressions"
[[package]]
name = "python-dateutil"
version = "2.8.2"
requires_python = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7"
summary = "Extensions to the standard Python datetime module"
dependencies = [
"six>=1.5",
]
[[package]]
name = "requests"
version = "2.27.1"
requires_python = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*"
summary = "Python HTTP for Humans."
dependencies = [
"certifi>=2017.4.17",
"charset-normalizer~=2.0.0; python_version >= \"3\"",
"idna<4,>=2.5; python_version >= \"3\"",
"urllib3<1.27,>=1.21.1",
]
[[package]]
name = "s3transfer"
version = "0.5.2"
requires_python = ">= 3.6"
summary = "An Amazon S3 Transfer Manager"
dependencies = [
"botocore<2.0a.0,>=1.12.36",
]
[[package]]
name = "six"
version = "1.16.0"
requires_python = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*"
summary = "Python 2 and 3 compatibility utilities"
[[package]]
name = "urllib3"
version = "1.26.9"
requires_python = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, <4"
summary = "HTTP library with thread-safe connection pooling, file post, and more."
[metadata]
lock_version = "3.1"
content_hash = "sha256:2cb85462fdf5d1e9853719d5dde86216f1809350cf43429c5b3659aa2b22f554"
[metadata.files]
"boto3 1.21.23" = [
{file = "boto3-1.21.23-py3-none-any.whl", hash = "sha256:c60d2401226c723db809df5c611a850c101b562e0e0790be7631a38b82813038"},
{file = "boto3-1.21.23.tar.gz", hash = "sha256:44a30268702c78aa355a0044fa7fea5f1e7e7c125ad029196028787eea003c2e"},
]
"botocore 1.24.46" = [
{file = "botocore-1.24.46-py3-none-any.whl", hash = "sha256:663d8f02b98641846eb959c54c840cc33264d5f2dee5b8fc09ee8adbef0f8dcf"},
{file = "botocore-1.24.46.tar.gz", hash = "sha256:89a203bba3c8f2299287e48a9e112e2dbe478cf67eaac26716f0e7f176446146"},
]
"certifi 2021.10.8" = [
{file = "certifi-2021.10.8-py2.py3-none-any.whl", hash = "sha256:d62a0163eb4c2344ac042ab2bdf75399a71a2d8c7d47eac2e2ee91b9d6339569"},
{file = "certifi-2021.10.8.tar.gz", hash = "sha256:78884e7c1d4b00ce3cea67b44566851c4343c120abd683433ce934a68ea58872"},
]
"charset-normalizer 2.0.12" = [
{file = "charset_normalizer-2.0.12-py3-none-any.whl", hash = "sha256:6881edbebdb17b39b4eaaa821b438bf6eddffb4468cf344f09f89def34a8b1df"},
{file = "charset-normalizer-2.0.12.tar.gz", hash = "sha256:2857e29ff0d34db842cd7ca3230549d1a697f96ee6d3fb071cfa6c7393832597"},
]
"idna 3.3" = [
{file = "idna-3.3-py3-none-any.whl", hash = "sha256:84d9dd047ffa80596e0f246e2eab0b391788b0503584e8945f2368256d2735ff"},
{file = "idna-3.3.tar.gz", hash = "sha256:9d643ff0a55b762d5cdb124b8eaa99c66322e2157b69160bc32796e824360e6d"},
]
"jmespath 1.0.0" = [
{file = "jmespath-1.0.0-py3-none-any.whl", hash = "sha256:e8dcd576ed616f14ec02eed0005c85973b5890083313860136657e24784e4c04"},
{file = "jmespath-1.0.0.tar.gz", hash = "sha256:a490e280edd1f57d6de88636992d05b71e97d69a26a19f058ecf7d304474bf5e"},
]
"python-dateutil 2.8.2" = [
{file = "python_dateutil-2.8.2-py2.py3-none-any.whl", hash = "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9"},
{file = "python-dateutil-2.8.2.tar.gz", hash = "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86"},
]
"requests 2.27.1" = [
{file = "requests-2.27.1-py2.py3-none-any.whl", hash = "sha256:f22fa1e554c9ddfd16e6e41ac79759e17be9e492b3587efa038054674760e72d"},
{file = "requests-2.27.1.tar.gz", hash = "sha256:68d7c56fd5a8999887728ef304a6d12edc7be74f1cfa47714fc8b414525c9a61"},
]
"s3transfer 0.5.2" = [
{file = "s3transfer-0.5.2-py3-none-any.whl", hash = "sha256:7a6f4c4d1fdb9a2b640244008e142cbc2cd3ae34b386584ef044dd0f27101971"},
{file = "s3transfer-0.5.2.tar.gz", hash = "sha256:95c58c194ce657a5f4fb0b9e60a84968c808888aed628cd98ab8771fe1db98ed"},
]
"six 1.16.0" = [
{file = "six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"},
{file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"},
]
"urllib3 1.26.9" = [
{file = "urllib3-1.26.9-py2.py3-none-any.whl", hash = "sha256:44ece4d53fb1706f667c9bd1c648f5469a2ec925fcf3a776667042d645472c14"},
{file = "urllib3-1.26.9.tar.gz", hash = "sha256:aabaf16477806a5e1dd19aa41f8c2b7950dd3c746362d7e3223dbe6de6ac448e"},
]
......@@ -4,9 +4,12 @@ version = "0.1.0"
description = ""
authors = [
{name = "Sam Woodcock", email = "samuel.woodcock@wsl.ch"},
{name= "Rebecca Kurup Buchholz"}
{name= "Rebecca Kurup Buchholz", email = "None"}
]
dependencies = [
"s3-bucket>=1.4.0",
"requests==2.27.1",
]
dependencies = ["boto3==1.21.23", "requests==2.27.1"]
requires-python = ">=3.9"
license = {text = "MIT"}
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment