Merge pull request #7 from cccb/staging-to-prod

WIP: Switch on new homepage in prod
This commit is contained in:
Daniel Molkentin 2023-10-25 00:30:08 +02:00 committed by GitHub
commit 0ca6400f2a
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
24 changed files with 450 additions and 261 deletions

19
.editorconfig Normal file
View file

@ -0,0 +1,19 @@
# EditorConfig is awesome: https://EditorConfig.org
# top-most EditorConfig file
root = true
# Unix-style newlines with a newline ending every file
[*]
charset = utf-8
end_of_line = lf
insert_final_newline = true
[*.py]
indent_style = space
indent_size = 4
[*.yaml]
indent_style = space
indent_size = 2

1
.github/CODEOWNERS vendored Normal file
View file

@ -0,0 +1 @@
* @cccb/web

28
.github/dependabot.yml vendored Normal file
View file

@ -0,0 +1,28 @@
# https://docs.github.com/github/administering-a-repository/configuration-options-for-dependency-updates
version: 2
updates:
- package-ecosystem: "github-actions"
directory: "/"
target-branch: "staging"
schedule:
interval: "weekly"
commit-message:
prefix: "gh-action"
labels:
- "gh-action"
- "dependencies"
reviewers:
- "cccb/web"
- package-ecosystem: "pip"
directory: "/"
target-branch: "dev"
schedule:
interval: "weekly"
commit-message:
prefix: "python"
labels:
- "python"
- "dependencies"
reviewers:
- "cccb/web"

126
.github/workflows/release.yml vendored Normal file
View file

@ -0,0 +1,126 @@
name: Release website
on:
push:
branches:
- staging
- production
pull_request:
workflow_dispatch:
jobs:
pages:
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v3
with:
submodules: recursive
fetch-depth: 0
- name: Setup Hugo
uses: peaceiris/actions-hugo@v2
with:
hugo-version: 'latest'
- name: Build pages
run: hugo $(cat .hugo-params)
- uses: actions/upload-artifact@v3
name: Upload pages
with:
name: pages
path: public
calendar:
needs: [ pages ]
runs-on: ubuntu-latest
steps:
- name: Add de_DE.UTF-8 locale
run: |
sudo apt-get update
sudo apt-get -y install locales
sudo locale-gen de_DE.UTF-8
- name: Checkout
uses: actions/checkout@v3
with:
submodules: recursive
fetch-depth: 0
- name: Setup Python
uses: actions/setup-python@v4
with:
cache: 'pip' # caching pip dependencies
- name: Install dependencies
run: |
pip install --upgrade pip setuptools wheel
pip install -r requirements.txt
- name: Download pages
uses: actions/download-artifact@v3
with:
name: pages
path: public/
- name: Generate calendars
run: python tools/merge_cals.py
- name: Copy calendar to output dir
run: cp static/all.ics public/all.ics
- name: Update homepage with latest event
run: upcoming="$(python tools/gen_upcoming.py static/all.ics 20 5 | tr '\n' ' ')" && sed -i "s#CALENDAR#$upcoming#g" public/index.html
- uses: actions/upload-artifact@v3
name: Upload pages
with:
name: enhanced_pages
path: public
staging:
needs: [ calendar ]
runs-on: ubuntu-latest
environment: staging
if: github.ref == 'refs/heads/staging' && github.event_name == 'push'
steps:
- name: Download pages
uses: actions/download-artifact@v3
with:
name: enhanced_pages
path: public
- name: Generate timestamp
run: echo "timestamp=$(date -u +'%Y-%m-%dT%H%M%SZ')" >> $GITHUB_ENV
- name: Create Release Archive
uses: thedoctor0/zip-release@0.7.1
with:
type: zip
filename: ../release-staging-${{ env.timestamp }}.zip
directory: public
- name: Create Release
uses: ncipollo/release-action@v1.12.0
with:
tag: staging-${{ env.timestamp }}
name: Website staging version ${{ env.timestamp }}
body: Website staging version ${{ env.timestamp }}
artifacts: release-staging-${{ env.timestamp }}.zip
token: ${{ secrets.GITHUB_TOKEN }}
production:
needs: [ calendar ]
runs-on: ubuntu-latest
environment: production
if: github.ref == 'refs/heads/production' && github.event_name == 'push'
steps:
- name: Download pages
uses: actions/download-artifact@v3
with:
name: enhanced_pages
path: public
- name: Generate timestamp
run: echo "timestamp=$(date -u +'%Y-%m-%dT%H%M%SZ')" >> $GITHUB_ENV
- name: Create Release Archive
uses: thedoctor0/zip-release@0.7.1
with:
type: zip
filename: ../release-production-${{ env.timestamp }}.zip
directory: public
- name: Create Release
uses: ncipollo/release-action@v1.12.0
with:
makeLatest: true
tag: production-${{ env.timestamp }}
name: Website production version ${{ env.timestamp }}
body: Website production version ${{ env.timestamp }}
artifacts: release-production-${{ env.timestamp }}.zip
token: ${{ secrets.GITHUB_TOKEN }}

94
.gitignore vendored
View file

@ -1,2 +1,94 @@
/public
static/all.ics
# Created by https://www.toptal.com/developers/gitignore/api/windows,linux,macos,hugo
# Edit at https://www.toptal.com/developers/gitignore?templates=windows,linux,macos,hugo
### Hugo ###
# Generated files by hugo
/public/
/resources/_gen/
/assets/jsconfig.json
hugo_stats.json
# Executable may be added to repository
hugo.exe
hugo.darwin
hugo.linux
# Temporary lock file while building
/.hugo_build.lock
### Linux ###
*~
# temporary files which can be created if a process still has a handle open of a deleted file
.fuse_hidden*
# KDE directory preferences
.directory
# Linux trash folder which might appear on any partition or disk
.Trash-*
# .nfs files are created when an open file is removed but is still being accessed
.nfs*
### macOS ###
# General
.DS_Store
.AppleDouble
.LSOverride
# Icon must end with two \r
Icon
# Thumbnails
._*
# Files that might appear in the root of a volume
.DocumentRevisions-V100
.fseventsd
.Spotlight-V100
.TemporaryItems
.Trashes
.VolumeIcon.icns
.com.apple.timemachine.donotpresent
# Directories potentially created on remote AFP share
.AppleDB
.AppleDesktop
Network Trash Folder
Temporary Items
.apdisk
### macOS Patch ###
# iCloud generated files
*.icloud
### Windows ###
# Windows thumbnail cache files
Thumbs.db
Thumbs.db:encryptable
ehthumbs.db
ehthumbs_vista.db
# Dump file
*.stackdump
# Folder config file
[Dd]esktop.ini
# Recycle Bin used on file shares
$RECYCLE.BIN/
# Windows Installer files
*.cab
*.msi
*.msix
*.msm
*.msp
# Windows shortcuts
*.lnk
# End of https://www.toptal.com/developers/gitignore/api/windows,linux,macos,hugo

View file

@ -1,93 +0,0 @@
stages:
- pages
- calendar
- deploy
cache:
paths:
- ~/.cache/pip/
- public/
variables:
GIT_SUBMODULE_STRATEGY: recursive
build_pages:
image: "golang:1.10-alpine3.8"
stage: pages
variables:
SHELL: "/bin/sh"
artifacts:
expire_in: "1 week"
untracked: true
script:
- rm -rf public/*
- apk add --no-cache --upgrade hugo
- hugo $(cat .hugo-params)
build_calendar:
image: "python:3.11.1-alpine3.17"
stage: calendar
dependencies:
- build_pages
artifacts:
expire_in: "1 week"
untracked: true
variables:
SHELL: "/bin/sh"
script:
- apk --no-cache update
- pip install -r requirements.txt
- python tools/merge_cals.py
- cp static/all.ics public/all.ics
- upcoming="$(python tools/gen_upcoming.py static/all.ics 20 5|tr '\n' ' ')" && sed -i "s#CALENDAR#$upcoming#g" public/index.html
deploy_staging:
image: "alpine:3.17"
stage: deploy
dependencies:
- build_calendar
variables:
SHELL: "/bin/sh"
script:
- apk --no-cache --upgrade add openssh-client rsync
- mkdir -p ~/.ssh
- eval $(ssh-agent -s)
- echo -e "Host *\n\tStrictHostKeyChecking no\n\n" > ~/.ssh/config
- echo "$PRIVATE_KEY" | ssh-add -
- rsync -e "ssh -l deploy -p 31337" -av --delete public/ 195.160.173.9:staging
when: on_success
environment:
name: staging
url: https://staging.berlin.ccc.de/
artifacts:
expire_in: "1 week"
paths:
- public/
only:
- staging
deploy_production:
image: "alpine:3.17"
stage: deploy
dependencies:
- build_calendar
variables:
SHELL: "/bin/sh"
script:
- apk --no-cache --upgrade add openssh-client rsync
- mkdir -p ~/.ssh
- eval $(ssh-agent -s)
- echo -e "Host *\n\tStrictHostKeyChecking no\n\n" > ~/.ssh/config
- echo "$PRIVATE_KEY" | ssh-add -
- rsync -e "ssh -l deploy -p 31337" -av --delete public/ 195.160.173.9:production
when: on_success
environment:
name: production
url: https://berlin.ccc.de/
artifacts:
expire_in: "1 week"
paths:
- public/
only:
- production

View file

@ -1 +1 @@
-b https://berlin.ccc.de/
--baseURL=https://berlin.ccc.de/

1
.python-version Normal file
View file

@ -0,0 +1 @@
3.11

View file

@ -1,45 +1,51 @@
[![pipeline status](https://gitlab.berlin.ccc.de/cccb/www/badges/master/pipeline.svg)](https://gitlab.berlin.ccc.de/cccb/www/commits/master)
![CCCB logo](static/img/logo.png)
# CCCB Website
This is the website of CCCB.
This is the website of the CCCB.
## Getting started
Get Hugo: <https://gohugo.io/getting-started/installing>
Clone this repo
```
git clone https://github.com/cccb/www
```
Switch directory
```
cd www
```
Fetch Submodules
```
git submodule update --recursive --remote --init
```
1. Get Hugo: <https://gohugo.io/getting-started/installing>
2. Clone this repo
```shell
git clone https://github.com/cccb/www
```
3. Switch directory
```shell
cd www
```
3. Fetch Submodules
```shell
git submodule update --recursive --remote --init
```
### Run site locally
Run hugo webserver
```
Run hugo webserver:
```shell
hugo serve
```
Point your browser to http://localhost:1313/
To ready your site for upload, run "./build.sh", which also generates all.ics and adds the calendar table to index.html
Point your browser to: http://localhost:1313/
To ready your site for upload, run `./build.sh`, which also generates `all.ics` and adds the calendar table to `index.html`.
Every change you make on the project will be reflected in your browser as long as `hugo serve` is running.
## Making a change
* Use your local dev setup (see Getting started) or via GitLab editor.
* Make your change in `staging` branch.
* Commit (and push) your change.
* Gitlab CI is running pipeline. If successfull, check [Staging Website](https://staging.berlin.ccc.de/) if change is correct.
* Create merge request to merge changes from `staging` to `production`. Ask somebody to check merge request or if small change, merge yourself.
* Gitlab CI is running pipeline. If successfull, check [Website](https://berlin.ccc.de/) if change is correct.
1. Use your local dev setup (see Getting started) or via GitHub editor.
2. Make your change in `staging` branch.
3. Commit (and push) your change.
4. GitHub Actions is running the release workflow.
- If successful, check [Staging Website](https://staging.berlin.ccc.de/) if change is correct.
5. Create merge request to merge changes from `staging` to `production` branch. Ask somebody to check merge request or if small change, merge yourself.
6. GitHub Actions is running the release workflow.
- If successfull, check [Website](https://berlin.ccc.de/) if change is correct.
7. Profit!
---
Made with ❤️ and [Hugo](https://gohugo.io).

13
TODO.md
View file

@ -1,15 +1,15 @@
Todo
----
# Todo
- DSGVO-compliant Datenschutzerklärung reinbasteln
- Entscheiden, welche Seiten sonst noch konvertiert werden sollen und welche in die ewigen Datengründe gehen können
- add nix config to repo
Done
----
# Done
- Template aussuchen
- Rausfinden, wie man eine Seite konvertiert
- hugo new page/mypage.md
- pandoc -f mediawiki page.mw -t markdown >> page/mypage.md
- `hugo new page/mypage.md`
- `pandoc -f mediawiki page.mw -t markdown >> page/mypage.md`
- Wichtigste Seiten konvertieren
- Membership-Seite auf den aktuellen Stand der Wahrheit bringen
- Ical template bauen
@ -17,3 +17,4 @@ Done
- Bestehende Datengarten-Termine konvertieren
- ggf. template mit frontmatter
- Theme forken, alle assets sollten lokal gehosted sein und nicht von irgendwelchen CDNs bezogen werden (HTTP/2 ftw!)

View file

@ -1,7 +1,8 @@
#!/usr/bin/env sh
#!/bin/sh
hugo $(cat .hugo-params)
tools/merge_cals.py
upcoming="$(tools/gen_upcoming.py static/all.ics 20 5|tr '\n' ' ')"
./tools/merge_cals.py
upcoming="$(tools/gen_upcoming.py static/all.ics 20 5 | tr '\n' ' ')"
cp static/all.ics public/all.ics
sed -i "s#CALENDAR#$upcoming#g" public/index.html

View file

@ -16,6 +16,8 @@ Params:
readingTime: true
useHLJS: true
DateForm: "30.12.2006"
# for GDPR / EU-DSGVO compliance
selfHosted: true
taxonomies:
category: "categories"
@ -63,7 +65,6 @@ mediaTypes:
suffixes:
- "xml"
outputFormats:
RSS:
mediaType: "application/rss"
@ -80,3 +81,4 @@ outputs:
page:
- "HTML"
- "Calendar"

View file

@ -11,7 +11,7 @@
{{ range .Params.tags }}
<a href="{{ $.Site.LanguagePrefix | absURL }}/tags/{{ . | urlize }}/">{{ . }}</a>&nbsp;
{{ end }}
</div>
</div><!--/.blog-tags-->
{{ end }}
{{ if .Site.Params.socialShare }}
@ -19,24 +19,24 @@
<section id="social-share">
<ul class="list-inline footer-links">
{{ partial "share-links" . }}
</ul>
</section>
</ul><!--/.social-share-->
</section><!--/.list-inline .footer-links-->
{{ end }}
</article>
</article><!--/.blog-post-->
{{ if ne .Type "page" }}
<ul class="pager blog-pager">
{{ if .PrevInSection }}
<li class="previous">
<a href="{{ .PrevInSection.Permalink }}" data-toggle="tooltip" data-placement="top" title="{{ .PrevInSection.Title }}">&larr; {{ i18n "previousPost" }}</a>
</li>
</li><!--/.previous-->
{{ end }}
{{ if .NextInSection }}
<li class="next">
<a href="{{ .NextInSection.Permalink }}" data-toggle="tooltip" data-placement="top" title="{{ .NextInSection.Title }}">{{ i18n "nextPost" }} &rarr;</a>
</li>
</li><!--/.next-->
{{ end }}
</ul>
</ul><!--/.pager .blog-pager-->
{{ end }}
@ -44,16 +44,16 @@
{{ if .Site.DisqusShortname }}
<div class="disqus-comments">
{{ template "_internal/disqus.html" . }}
</div>
</div><!--/.disqus-comments-->
{{ end }}
{{ if .Site.Params.staticman }}
<div class="staticman-comments">
{{ partial "staticman-comments.html" . }}
</div>
</div><!--/.staticman-comments-->
{{ end }}
{{ end }}
</div>
</div>
</div>
</div><!--/.col-lg-8 .col-lg-offset-2 .col-md-10 .col-md-offset-1-->
</div><!--/.row-->
</div><!--/.container-->
{{ end }}

View file

@ -1,12 +1,12 @@
{{ $series := or (.Get 0) $.Page.Params.series }}
<table>
<tr>
<th>No.</th>
<th>Date</th>
<th>Speaker</th>
<th>Topic</th>
<th>Video</th>
</tr>
<tr>
<th>No.</th>
<th>Date</th>
<th>Speaker</th>
<th>Topic</th>
<th>Video</th>
</tr>
{{ range $ind,$art := $.Site.Pages.ByDate.Reverse }}
{{ if eq $art.Params.series $series }}
<tr>

View file

@ -1 +1 @@
icalendar
icalendar==5.0.7

Binary file not shown.

Before

Width:  |  Height:  |  Size: 26 KiB

After

Width:  |  Height:  |  Size: 41 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 19 KiB

After

Width:  |  Height:  |  Size: 22 KiB

@ -1 +1 @@
Subproject commit 0da13d20d0bf3c9002651f06a54d21608d292958
Subproject commit 1e66e4ae945f12da3f8e6fb603c396156c1b04e9

View file

@ -1,15 +0,0 @@
#!/bin/bash
if [ $# -lt 2 ]; then
echo "Usage: $0 Mediawiki_Page_Name [page|post]"
exit
fi
echo Converting $1...
page=$(echo $1 | sed 's,/,_,g'|tr '[:upper:]' '[:lower:]')
hugo new $2/$page.md
curl -s https://berlin.ccc.de/api.php\?action\=query\&prop\=revisions\&rvprop\=content\&format\=json\&titles\=$1 | \
jq -r '.query.pages |..| objects|.["*"]'| sed '/null/d' | pandoc -f mediawiki -t markdown \
>> content/$2/$page.md

17
tools/convert_page.sh Executable file
View file

@ -0,0 +1,17 @@
#!/bin/sh
if [ $# -lt 2 ]; then
echo "Usage: $0 Mediawiki_Page_Name [page|post]"
exit
fi
echo Converting $1...
page=$(echo $1 | sed 's,/,_,g' | tr '[:upper:]' '[:lower:]')
hugo new $2/$page.md
curl -s "https://berlin.ccc.de/api.php?action=query&prop=revisions&rvprop=content&format=json&titles=$1" \
| jq -r '.query.pages |..| objects|.["*"]' \
| sed '/null/d' \
| pandoc -f mediawiki -t markdown \
>> content/$2/$page.md

View file

@ -3,7 +3,6 @@
import sys
import logging
import locale
from pprint import pprint
from dateutil.parser import parse
from datetime import datetime, timedelta
from dateutil.rrule import rruleset, rrulestr
@ -13,78 +12,82 @@ import icalendar
def vevent_to_event(event, rrstart=None):
if rrstart == None:
begin = parse(event['DTSTART'].to_ical())
begin = parse(event["DTSTART"].to_ical())
else:
begin = rrstart
return { "name": event['SUMMARY'].to_ical(), "url": event['URL'].to_ical(), "begin": begin }
return {
"name": event["SUMMARY"].to_ical(),
"url": event["URL"].to_ical(),
"begin": begin
}
def parse_single_event(event, start, end):
logging.info("Processing single event %s" % event['SUMMARY'].to_ical().decode('utf-8'))
dtstart = parse(event['DTSTART'].to_ical())
logging.info(f"Processing single event {event['SUMMARY'].to_ical().decode('utf-8')}")
dtstart = parse(event["DTSTART"].to_ical())
if dtstart >= start and dtstart < end:
return vevent_to_event(event)
else:
return None
def parse_recurring_event(event, start, end):
logging.info("Processing recurring event %s" % event['SUMMARY'].to_ical().decode('utf-8'))
dtstart = parse(event['DTSTART'].to_ical())
logging.info(f"Processing recurring event {event['SUMMARY'].to_ical().decode('utf-8')}")
dtstart = parse(event["DTSTART"].to_ical())
rs = rruleset()
rs.rrule(rrulestr(event['RRULE'].to_ical().decode('utf-8'), dtstart=dtstart))
if 'EXDATE' in event.keys():
exdates = event['EXDATE']
for exdate in exdates:
rs.rrule(rrulestr(event["RRULE"].to_ical().decode("utf-8"), dtstart=dtstart))
if "EXDATE" in event.keys():
for exdate in event["EXDATE"]:
rs.exdate(parse(exdate.to_ical()))
dates = list(rs)
events = []
for date in dates:
for date in list(rs):
if date >= start and date < end:
events.append(vevent_to_event(event, date))
return events
def find_events(icsfilestr, start, end, num):
with open(icsfilestr, 'r') as icsfile:
cal=icalendar.Calendar.from_ical(icsfile.read())
with open(icsfilestr, "r") as icsfile:
cal = icalendar.Calendar.from_ical(icsfile.read())
events=[]
events = []
for event in cal.subcomponents:
if event.name == 'VEVENT':
if 'RRULE' in event.keys():
events = events + parse_recurring_event(event, start, end)
else:
ev = parse_single_event(event, start, end)
if ev != None:
if event.name == "VEVENT":
if "RRULE" in event.keys():
events.extend(parse_recurring_event(event, start, end))
elif ev := parse_single_event(event, start, end) != None:
events.append(ev)
events = sorted(events, key=lambda k: k['begin'])
events = sorted(events, key=lambda k: k["begin"])
events = events[0:num]
return events
def format_events(events):
print('<table class="table table-condensed">')
print("<table class=\"table table-condensed\">")
for event in events:
dateStr = event['begin'].strftime("%A, %d.%m um %H:%M Uhr")
#print("<li><a href=\"%s\">%s: %s</a></li>" % (event['url'].decode('utf-8'), dateStr, event['name'].decode('utf-8')))
print("<tr><td>%s</td><td><a href=\"%s\">%s</a></td></tr>"
% (dateStr, event['url'].decode('utf-8'), event['name'].decode('utf-8')))
print('</table>')
print(
"<tr>"
f"<td>{event['begin'].strftime('%A, %d.%m um %H:%M Uhr')}</td>"
f"<td><a href=\"{event['url'].decode('utf-8')}\">{event['name'].decode('utf-8')}</a></td>"
"</tr>"
)
print("</table><!--/.table .table-condensed-->")
if __name__ == "__main__":
if len(sys.argv) < 3:
print("Usage: %s calendar max_days max_items" % sys.argv[0])
print(f"Usage: {sys.argv[0]} calendar max_days max_items")
sys.exit(-1)
locale.setlocale(locale.LC_TIME, "de_DE.UTF-8")
calendar=sys.argv[1]
max_days=int(sys.argv[2])
max_items=int(sys.argv[3])
calendar = sys.argv[1]
max_days = int(sys.argv[2])
max_items = int(sys.argv[3])
events=find_events(calendar, datetime.now(), datetime.now() + timedelta(days=max_days), max_items)
now = datetime.now()
events = find_events(calendar, now, now + timedelta(days=max_days), max_items)
format_events(events)

View file

@ -6,22 +6,22 @@ import pytz
import icalendar
cals = []
calendars = []
merged = icalendar.Calendar()
merged.add('prodid', '-//CCCB Calendar Generator//berlin.ccc.de//')
merged.add('version', '2.0')
merged.add("prodid", "-//CCCB Calendar Generator//berlin.ccc.de//")
merged.add("version", "2.0")
for icsfilestr in glob('public/*/**/*.ics', recursive=True):
with open(icsfilestr, 'r') as icsfile:
print('Importing', icsfilestr)
cals.append(icalendar.Calendar.from_ical(icsfile.read()))
for icsfilestr in glob("public/*/**/*.ics", recursive=True):
with open(icsfilestr, "r") as icsfile:
print(f"Importing {icsfilestr}")
calendars.append(icalendar.Calendar.from_ical(icsfile.read()))
for cal in cals:
for e in cal.subcomponents:
merged.add_component(e)
for calendar in calendars:
for event in calendar.subcomponents:
merged.add_component(event)
outfile = 'static/all.ics'
with open(outfile, 'wb') as f:
print(f'writing to {outfile}...')
outfile = "static/all.ics"
with open(outfile, "wb") as f:
print(f"writing to {outfile}...")
f.write(merged.to_ical())