diff --git a/README.md b/README.md index 13150c1..4695f6f 100644 --- a/README.md +++ b/README.md @@ -13,21 +13,82 @@ Output from this tool is shown on [the ActivityWatch website](https://activitywa - Generate tables from git history with number of active days, number of commits, and diff stats. - Generate statistics from GitHub activity (issues, comments, PRs). - - Create a video visualization, such as the one made for [ActivityWatch](). + - Create a video visualization, such as the one made for [ActivityWatch](http://www.youtube.com/watch?v=zjIn43lZq3U). ## Gource visualization This also includes scripts to produce a visualization of the commit history with [gource](https://gource.io/). -Usage: +### Usage -``` +```bash cd video ./gource-output.sh ``` -NOTE: It assumes you have the repos cloned with a certain directory structure. You will probably need to modify the script to suit your folder structure. +### Directory Structure + +The script assumes a specific directory layout relative to the video folder: + +```txt +../../../ # rootdir +├── activitywatch/ # Main bundle repo (cloned as activitywatch or .) +│ ├── aw-core/ +│ ├── aw-client/ +│ ├── aw-server/ +│ │ └── aw-webui/ +│ ├── aw-server-rust/ +│ ├── aw-qt/ +│ ├── aw-watcher-afk/ +│ ├── aw-watcher-window/ +│ └── ... +├── other/ # Other official repos +│ ├── aw-client-js/ +│ ├── aw-watcher-web/ +│ ├── aw-watcher-window-wayland/ +│ ├── aw-tauri/ +│ ├── aw-sync/ +│ ├── aw-notify/ +│ ├── activitywatch.github.io/ +│ ├── aw-research/ +│ ├── aw-watcher-vscode/ +│ └── aw-watcher-vim/ +├── community/ # Community-contributed projects +│ ├── awatcher/ # https://github.com/2e3s/awatcher +│ ├── aw-watcher-media-player/ +│ ├── aw-watcher-jetbrains/ +│ └── activitywatch-plasmoid/ +├── docs/ +└── old/ + └── activitywatch-old/ +``` + +### Including Community Repos + +To include community projects in the visualization: + +1. Create a `community/` directory next to the main repos +2. Clone the community repos you want to include: + +```bash +mkdir -p community +cd community +git clone https://github.com/2e3s/awatcher +git clone https://github.com/2e3s/aw-watcher-media-player +git clone https://github.com/OlivierMary/aw-watcher-jetbrains +git clone https://github.com/NicoWeio/activitywatch-plasmoid +``` + +The script will automatically skip repos that aren't found, so you can include as many or as few community repos as desired. ### Output [![Example of visualization rendered with gource](http://img.youtube.com/vi/zjIn43lZq3U/0.jpg)](http://www.youtube.com/watch?v=zjIn43lZq3U "ActivityWatch Development Visualization 2014-2020, with Gource") + +### Adding music + +After generating `gource.mp4`, you can add music with: + +```bash +./gource-output-add-music.sh +``` diff --git a/video/gource-output.sh b/video/gource-output.sh index a606b66..74946de 100755 --- a/video/gource-output.sh +++ b/video/gource-output.sh @@ -12,6 +12,9 @@ echo "Building stuff" # Bundle repo gource --output-custom-log $tmpdir/activitywatch.txt $rootdir +# =========================================== +# Official ActivityWatch modules +# =========================================== modules=( aw-core # clients @@ -33,11 +36,34 @@ modules=( docs # misc other/aw-research + # experimental + other/aw-tauri + # sync and notifications + other/aw-sync + other/aw-notify # old old/activitywatch-old # hidden due it causing a mess #other/aw-android ) + +# =========================================== +# Community-contributed projects +# These are popular community projects from awesome-activitywatch +# Clone them to $rootdir/community/ before running +# =========================================== +community_modules=( + # Watchers + community/awatcher # Popular X11/Wayland watcher by @2e3s + community/aw-watcher-media-player # Media playback watcher by @2e3s + # Editor integrations + other/aw-watcher-vscode # VSCode extension + other/aw-watcher-vim # Vim extension + community/aw-watcher-jetbrains # JetBrains IDEs + # Desktop widgets + community/activitywatch-plasmoid # KDE Plasma widget by @NicoWeio +) + for path in "${modules[@]}"; do name=$(basename $path) loc=$(echo $name | sed -E "s#.+-watcher-.+#watchers/$name#g") @@ -45,13 +71,37 @@ for path in "${modules[@]}"; do loc=$(echo $loc | sed -E "s#.+-server.*#servers/$name#g") loc=$(echo $loc | sed -E "s#docs|activitywatch.github.io#website/$name#g") echo $name $loc - gource --output-custom-log $tmpdir/$name.txt $rootdir/$path - sed -i -r "s#(.+)\\|#\\1|/$loc#" $tmpdir/$name.txt + if [ -d "$rootdir/$path" ]; then + gource --output-custom-log $tmpdir/$name.txt $rootdir/$path + sed -i -r "s#(.+)\\|#\\1|/$loc#" $tmpdir/$name.txt + else + echo " -> Skipping (not found): $rootdir/$path" + fi +done + +# Process community modules +for path in "${community_modules[@]}"; do + name=$(basename $path) + # Categorize community modules + if [[ $name == *"watcher"* ]] || [[ $name == "awatcher" ]]; then + loc="watchers/community/$name" + elif [[ $name == *"plasmoid"* ]] || [[ $name == *"widget"* ]]; then + loc="widgets/$name" + else + loc="community/$name" + fi + echo "$name -> $loc (community)" + if [ -d "$rootdir/$path" ]; then + gource --output-custom-log $tmpdir/$name.txt $rootdir/$path + sed -i -r "s#(.+)\\|#\\1|/$loc#" $tmpdir/$name.txt + else + echo " -> Skipping (not found): $rootdir/$path" + fi done # Remove all files in activitywatch-old repo when rewrite began # TODO: Remove in a logical order (deepest first) -sed -E 's/.+[|](.+)[|].+[|](.+)/1461708000|\1|D|\2/g' $tmpdir/activitywatch-old.txt | uniq > $tmpdir/fixes.txt +sed -E 's/.+[|](.+)[|].+[|](.+)/1461708000|\1|D|\2/g' $tmpdir/activitywatch-old.txt 2>/dev/null | uniq > $tmpdir/fixes.txt || true gourcelog=$tmpdir/combined.gource @@ -86,6 +136,9 @@ sed -i 's/johan-bjareholt/Johan Bjäreholt/g' $gourcelog sed -i -E 's/Erik Bj.{1,4}reholt/Erik Bjäreholt/g' $gourcelog sed -i 's/dependabot.+/dependabot/g' $gourcelog sed -i 's/Bill-linux/Bill Ang Li/g' $gourcelog +# Community contributor name fixes +sed -i 's/2e3s/Denis Gavrilov/g' $gourcelog +sed -i 's/NicoWeio/Nico Weißenbacher/g' $gourcelog # Remove names which have been spamming commits in CI (accidental bad CI config) sed -i 's/.*ErikBjare.*//g' $gourcelog @@ -93,17 +146,17 @@ sed -i 's/.*ErikBjare.*//g' $gourcelog # Prepare avatars # TODO: Doesn't fetch avatars from all repos (only the ones with most contributors) # run for contributor-stats repo, initialized .git/avatars folder -if [ -x .git/avatars ]; then +if [ -d .git/avatar ]; then perl fetch-avatars.pl # run for bundle repo, move avatars to local .git/avatars fetchsrc=$(realpath fetch-avatars.pl) - pushd $rootdir; perl $fetchsrc; popd; mv $rootdir/.git/avatar/* .git/avatar - pushd $rootdir/aw-server/aw-webui; perl $fetchsrc; popd; mv $rootdir/aw-server/aw-webui/.git/avatar/* .git/avatar - pushd $rootdir/docs; perl $fetchsrc; popd; mv $rootdir/docs/.git/avatar/* .git/avatar + pushd $rootdir; perl $fetchsrc; popd; mv $rootdir/.git/avatar/* .git/avatar 2>/dev/null || true + pushd $rootdir/aw-server/aw-webui; perl $fetchsrc; popd; mv $rootdir/aw-server/aw-webui/.git/avatar/* .git/avatar 2>/dev/null || true + pushd $rootdir/docs; perl $fetchsrc; popd; mv $rootdir/docs/.git/avatar/* .git/avatar 2>/dev/null || true fi # Rename avatars to suit committer name -cp ../.git/avatar/johan-bjareholt.png '../.git/avatar/Johan Bjäreholt.png' +[ -f ../.git/avatar/johan-bjareholt.png ] && cp ../.git/avatar/johan-bjareholt.png '../.git/avatar/Johan Bjäreholt.png' # Resolutions: # - 2560x1440 (for upload) diff --git a/video/issue-history-export.py b/video/issue-history-export.py new file mode 100755 index 0000000..9ee27a6 --- /dev/null +++ b/video/issue-history-export.py @@ -0,0 +1,217 @@ +#!/usr/bin/env python3 +""" +Export GitHub issue history for ActivityWatch repos. + +This script fetches issue activity (creation, comments, closure) from GitHub +and exports it in a format suitable for visualization. + +Usage: + ./issue-history-export.py [--output issues.csv] [--format csv|gource] + +Output formats: + csv: Standard CSV with timestamp, repo, issue, event_type, author + gource: Gource-compatible log format for visualization +""" + +import subprocess +import json +import sys +from datetime import datetime +from pathlib import Path + +# ActivityWatch repos to include +REPOS = [ + "ActivityWatch/activitywatch", + "ActivityWatch/aw-core", + "ActivityWatch/aw-server", + "ActivityWatch/aw-server-rust", + "ActivityWatch/aw-qt", + "ActivityWatch/aw-webui", + "ActivityWatch/aw-watcher-afk", + "ActivityWatch/aw-watcher-window", + "ActivityWatch/aw-watcher-web", + "ActivityWatch/docs", + "ActivityWatch/activitywatch.github.io", +] + +def fetch_issues(repo: str, max_issues: int = 500) -> list: + """Fetch issues and their timeline from a GitHub repo.""" + print(f"Fetching issues from {repo}...", file=sys.stderr) + + # Fetch issues with comments count + cmd = [ + "gh", "api", "-X", "GET", + f"/repos/{repo}/issues", + "-f", "state=all", + "-f", "per_page=100", + "--paginate", + "--jq", """ + .[] | { + number: .number, + title: .title, + state: .state, + created_at: .created_at, + closed_at: .closed_at, + user: .user.login, + comments: .comments, + is_pr: (.pull_request != null) + } + """ + ] + + try: + result = subprocess.run(cmd, capture_output=True, text=True, timeout=60) + if result.returncode != 0: + print(f" Warning: Failed to fetch {repo}: {result.stderr}", file=sys.stderr) + return [] + + issues = [] + for line in result.stdout.strip().split('\n'): + if line: + try: + issues.append(json.loads(line)) + except json.JSONDecodeError: + pass + + return issues[:max_issues] + except subprocess.TimeoutExpired: + print(f" Warning: Timeout fetching {repo}", file=sys.stderr) + return [] + +def fetch_issue_comments(repo: str, issue_number: int) -> list: + """Fetch comments for a specific issue.""" + cmd = [ + "gh", "api", "-X", "GET", + f"/repos/{repo}/issues/{issue_number}/comments", + "--jq", '.[] | {created_at: .created_at, user: .user.login}' + ] + + try: + result = subprocess.run(cmd, capture_output=True, text=True, timeout=30) + if result.returncode != 0: + return [] + + comments = [] + for line in result.stdout.strip().split('\n'): + if line: + try: + comments.append(json.loads(line)) + except json.JSONDecodeError: + pass + return comments + except subprocess.TimeoutExpired: + return [] + +def to_timestamp(iso_date: str) -> int: + """Convert ISO date string to Unix timestamp.""" + if not iso_date: + return 0 + try: + dt = datetime.fromisoformat(iso_date.replace('Z', '+00:00')) + return int(dt.timestamp()) + except: + return 0 + +def export_gource_format(events: list) -> str: + """Convert events to Gource-compatible log format. + + Gource format: timestamp|author|action|path + Actions: A (add), M (modify), D (delete) + """ + lines = [] + for event in sorted(events, key=lambda e: e['timestamp']): + # Map event types to gource actions + if event['type'] == 'created': + action = 'A' # Add = issue opened + elif event['type'] == 'comment': + action = 'M' # Modify = comment added + elif event['type'] == 'closed': + action = 'D' # Delete = issue closed + else: + action = 'M' + + # Create path like: repo/issues/issue-number + path = f"{event['repo']}/issues/{event['issue_number']}" + + lines.append(f"{event['timestamp']}|{event['author']}|{action}|{path}") + + return '\n'.join(lines) + +def export_csv_format(events: list) -> str: + """Convert events to CSV format.""" + lines = ["timestamp,datetime,repo,issue_number,type,author,title"] + for event in sorted(events, key=lambda e: e['timestamp']): + dt = datetime.fromtimestamp(event['timestamp']).isoformat() + title = event.get('title', '').replace(',', ' ').replace('"', "'")[:50] + lines.append(f"{event['timestamp']},{dt},{event['repo']},{event['issue_number']},{event['type']},{event['author']},{title}") + return '\n'.join(lines) + +def main(): + import argparse + parser = argparse.ArgumentParser(description="Export GitHub issue history") + parser.add_argument('--output', '-o', default='issues.csv', help='Output file') + parser.add_argument('--format', '-f', choices=['csv', 'gource'], default='csv') + parser.add_argument('--include-comments', action='store_true', help='Include comment events (slower)') + args = parser.parse_args() + + all_events = [] + + for repo in REPOS: + issues = fetch_issues(repo) + print(f" Found {len(issues)} issues in {repo}", file=sys.stderr) + + for issue in issues: + if issue.get('is_pr'): + continue # Skip PRs for now + + repo_short = repo.split('/')[-1] + + # Issue created event + all_events.append({ + 'timestamp': to_timestamp(issue['created_at']), + 'repo': repo_short, + 'issue_number': issue['number'], + 'type': 'created', + 'author': issue['user'], + 'title': issue['title'] + }) + + # Issue closed event + if issue.get('closed_at'): + all_events.append({ + 'timestamp': to_timestamp(issue['closed_at']), + 'repo': repo_short, + 'issue_number': issue['number'], + 'type': 'closed', + 'author': issue['user'], # Could fetch closer + 'title': issue['title'] + }) + + # Fetch comments if requested (slower due to API rate limits) + if args.include_comments and issue.get('comments', 0) > 0: + comments = fetch_issue_comments(repo, issue['number']) + for comment in comments: + all_events.append({ + 'timestamp': to_timestamp(comment['created_at']), + 'repo': repo_short, + 'issue_number': issue['number'], + 'type': 'comment', + 'author': comment['user'], + 'title': issue['title'] + }) + + print(f"\nTotal events: {len(all_events)}", file=sys.stderr) + + # Export in requested format + if args.format == 'gource': + output = export_gource_format(all_events) + else: + output = export_csv_format(all_events) + + # Write output + output_path = Path(args.output) + output_path.write_text(output) + print(f"Wrote {len(all_events)} events to {output_path}", file=sys.stderr) + +if __name__ == '__main__': + main()