initial commit

.githooks/pre-commit

@@ -0,0 +1,50 @@
+#!/usr/bin/env bash
+
+set -o errexit
+set -o nounset
+
+fail() {
+  echo "$1" >&2
+  exit 1
+}
+
+preflight() {
+  local cmd=$1
+  local msg="%s not found: install it or use the --no-verify flag for the git commit to avoid the running this pre-commit hook.\n"
+  command -v $cmd >/dev/null 2>&1 || fail \
+    "$(printf "$msg" $cmd)"
+}
+
+#
+# preflight checks
+#
+preflight terraform
+preflight yq
+
+# terraform fmt -check
+tf_files=($(git diff --cached --name-only --diff-filter=ACM -- '*.tf' '*.tfvars'))
+if [ ${#tf_files[@]} -gt 0 ]; then
+  # terraform fmt -check returns a non-zero exit status and list of files to be
+  # formatted: override that exit status so this entire script doesn't exit prematurely
+  output=$(printf '%s\n' "${tf_files[@]}" | xargs terraform fmt -check 2>&1 || true)
+  if [ -n "$output" ]; then
+    printf 'The following files are incorrectly formated: run `terraform fmt` to fix them.\n' >&2
+    fail $output
+  fi
+fi
+
+# yq eval                                                                                                                                                                  
+yaml_files=($(git diff --cached --name-only --diff-filter=ACM -- '*.yml' '*.yaml'))
+if [ ${#yaml_files[@]} -gt 0 ]; then
+  broken_files=()
+  for f in ${yaml_files[@]}; do
+    yq --exit-status 'tag == "!!map" or tag== "!!seq"' >/dev/null 2>&1 $f || broken_files+=($f)
+  done 
+  if [ ${#broken_files[@]} -gt 0 ]; then
+    echo 'The following files appear to contain invalid yaml:' >&2
+    fail "$(printf '%s\n' ${broken_files[@]})"
+#    echo 'The following files appear to contain invalid yaml:' >&2
+#  	printf '%s\n' ${broken_files[@]} >&2
+#	  exit 1
+  fi
+fi

.gitignore

@@ -0,0 +1 @@
+.env

README.md

@@ -0,0 +1,9 @@
+# taskfiles
+
+Here are some [taskfiles](https://taskfile.dev/) I use.
+
+## Hooks
+Git hooks for this repo are provided. To use them, configure the `hooksPath` for the repo:
+```
+$ git config --local core.hooksPath .githooks
+```

Taskfile.yml

@@ -0,0 +1,11 @@
+version: '3'
+
+includes:
+  aws: ./cfa-taskfiles/aws.yml
+  int: ./cfa-taskfiles/integrations.yml
+  mobile: ./cfa-taskfiles/mobile.yml
+  k8s: ./cfa-taskfiles/k8s.yml
+  grafana: ./cfa-taskfiles/grafana.yml
+  pyrenees: ./cfa-taskfiles/pyrenees.yml
+  spacelift: ./cfa-taskfiles/spacelift.yml
+

bootstrap/Taskfile-alpine.yml

@@ -0,0 +1,29 @@
+version: '3'
+
+tasks:
+  default:
+    desc: initialize system
+    cmds:
+      - task: packages
+      - task: dotfiles
+      - task: misc
+
+  packages:
+    desc: install packages
+    cmd: |
+      sudo /sbin/apk -U add \
+        tmux zsh zsh-vcs git neovim py3-pynvim stow fzf \
+        > /dev/null
+
+  dotfiles:
+    desc: install my dotfiles
+    dir: $HOME
+    cmds:
+      - test -d dotfiles || git clone https://git.scalene.net/jswank/dotfiles.git > /dev/null
+      - cd dotfiles && ./install.sh > /dev/null
+
+  misc:
+    cmds:
+      - chsh -s /bin/zsh
+      - mkdir -p $HOME/.local/bin
+      - curl -Ls https://github.com/marwanhawari/stew/releases/download/v0.4.0/stew-v0.4.0-linux-amd64.tar.gz | tar -xz stew -C $HOME/.local/bin

bootstrap/Taskfile-ubuntu.yml

@@ -0,0 +1,28 @@
+version: '3'
+
+tasks:
+  default:
+    desc: initialize system
+    cmds:
+      - task: packages
+      - task: dotfiles
+      - task: misc
+
+  packages:
+    desc: install packages
+    cmds:
+      - sudo /usr/bin/apt update --yes
+      - sudo /usr/bin/apt install --yes tmux zsh git neovim stow fzf
+
+  dotfiles:
+    desc: install my dotfiles
+    dir: $HOME
+    cmds:
+      - test -d dotfiles || git clone https://git.scalene.net/jswank/dotfiles.git > /dev/null
+      - cd dotfiles && ./install.sh > /dev/null
+
+  misc:
+    cmds:
+      - sudo chsh -s $(which zsh) $(whoami)
+      - mkdir -p $HOME/.local/bin
+      - curl -Ls https://github.com/marwanhawari/stew/releases/download/v0.4.0/stew-v0.4.0-linux-amd64.tar.gz | tar -xz stew -C $HOME/.local/bin

bootstrap/Taskfile.yml

@@ -0,0 +1,26 @@
+version: '3'
+
+tasks:
+  default:
+    desc: initialize system
+    cmds:
+      - task: packages
+      - task: dotfiles
+      - task: misc
+
+  packages:
+    desc: install packages
+    cmds:
+      - sudo /usr/bin/apt update --yes
+      - sudo /usr/bin/apt install --yes tmux zsh git neovim stow fzf
+
+  dotfiles:
+    desc: install my dotfiles
+    dir: $HOME
+    cmds:
+      - test -d dotfiles || git clone https://git.scalene.net/jswank/dotfiles.git > /dev/null
+      - cd dotfiles && ./install.sh > /dev/null
+
+  misc:
+    cmds:
+      - sudo chsh -s $(which zsh) $(whoami)

bootstrap/bootstrap-ubuntu

@@ -0,0 +1,14 @@
+#!/usr/bin/env bash
+
+set -o nounset
+set -o errexit
+
+cd ~/
+
+mkdir -p .local/bin
+
+if [[ ! -x ~/.local/bin/task ]]; then
+  sh -c "$(curl --silent --location https://taskfile.dev/install.sh)" -- -d -b ~/.local/bin 
+fi
+
+~/.local/bin/task 

cfa-taskfiles/aws.yml

@@ -0,0 +1,99 @@
+version: '3'
+
+tasks:
+  subshell:
+    desc: invoke a subshell w/ correct AWS variables set
+    cmds:
+      - cmd: |
+          eval `okta-aws-cli web`; AWS_ENVIRONMENT="$(aws iam list-account-aliases | jq -r '.AccountAliases[0]')" zsh
+      - cmd: rm -f {{ .USER_WORKING_DIR }}/.session-env
+  logout:
+    desc: Invalidate the OKTA credential
+    cmd: rm -f ${HOME}/.okta/awscli-access-token.json 
+    preconditions:
+      - test -f ${HOME}/.okta/awscli-access-token.json
+  list-secrets:
+    desc: List all secrets
+    cmd: |
+      aws secretsmanager list-secrets | jq '[.SecretList[] | {"name" :.Name, "arn": .ARN, "desc": .Description}]'
+  get-secret:
+    desc: Get a secret - supply the name the secret as arg1
+    cmd: |
+      aws secretsmanager get-secret-value --secret-id {{.CLI_ARGS}} --query SecretString --output text
+  list-load-balancers:
+    desc: List all load balancers
+    cmd: aws elbv2 describe-load-balancers | jq -r '.LoadBalancers[].LoadBalancerArn'
+  list-listeners:
+    desc: List all listners for a load balancer - supply the load balancer ARN as arg1
+    cmd: |
+      aws elbv2 describe-listeners --load-balancer-arn {{.CLI_ARGS}} | jq -r '.Listeners[].ListenerArn'
+  get-rules:
+    desc: get rules - supply listener ARN as arg1
+    cmd: |
+      aws elbv2 describe-rules --listener-arn {{.CLI_ARGS}} | gron |grep HostHeaderConfig.Values |grep -v '];'
+#  get-policy:
+#    desc: return the latest version of the specified policy
+#    cmd: |
+#    vars:
+#      POLICY_NAME:
+#      POLICY_ID:
+  list-clusters:
+    desc: list EKS clusters
+    cmd: aws eks list-clusters | jq -r '.clusters[]'
+  kubeconfig:
+    desc: update kubeconfig for the given name
+    cmd: aws eks update-kubeconfig --name {{.CLI_ARGS}}
+  infer-kubeconfig:
+    desc: infer kubeconfig based on ETS SRE conventions
+      # cmd: aws eks update-kubeconfig --name {{.CLI_ARGS}}
+    cmd: aws eks update-kubeconfig --name {{.CLUSTER_NAME}}
+    vars:
+      CLUSTER_NAME:
+        sh: aws eks list-clusters | jq -r '.clusters[0]'
+  connect-alloy-pod:
+    desc: run a shell on an alloy pod
+    cmd: kubectl exec -it alloy-6qflx --namespace grafana -- /bin/bash
+  forward-alloy-pod:
+    desc: port forward an alloy pod
+    cmd: kubectl port-forward alloy-6qflx --address 0.0.0.0 12345:12345 --namespace grafana
+  infer-alb-hosts:
+    desc: infer hostnames supported by an ALB based on ETS SRE conventions
+    silent: true
+    cmd: |
+      #!/bin/sh
+      set -o errexit
+      set -o pipefail
+      alb_arn=$(aws elbv2 describe-load-balancers | jq -r '.LoadBalancers[].LoadBalancerArn' | grep awsingress)
+      listener_arn=$(aws elbv2 describe-listeners --load-balancer-arn $alb_arn | jq -r '.Listeners[].ListenerArn')
+      aws elbv2 describe-rules --listener-arn "$listener_arn" | gron | grep 'HostHeaderConfig.Values\[' | perl -nE 'say $1 if m/"(.+)"/'
+  get-spacelift-runs:
+    desc: return a list of all spacelift runs in the last 4 hours
+    cmd: |
+      aws cloudtrail lookup-events \
+        --lookup-attributes AttributeKey=EventName,AttributeValue=AssumeRoleWithWebIdentity \
+        --start-time "$(date -u -d '-240 minutes' '+%Y-%m-%dT%H:%M:%SZ')" \
+        --end-time "$(date -u '+%Y-%m-%dT%H:%M:%SZ')" \
+          | jq -r '.Events[]
+            | select(.Resources[].ResourceName
+            | endswith(":role/spacelift"))
+            | .Resources[]
+            | select(.ResourceType == "AWS::STS::AssumedRole" and (.ResourceName | type == "string" and startswith("spacelift-run")))
+            | .ResourceName' 
+  get-spacelift-run:
+    desc: get all events for a specific spacelift RUN id which occured in the last 24 hours
+    cmd: | 
+      aws cloudtrail lookup-events \
+        --lookup-attributes AttributeKey=Username,AttributeValue=spacelift-run-{{ .RUN }} \
+        --start-time "$(date -u -d '-24 hours' '+%Y-%m-%dT%H:%M:%SZ')" \
+        --end-time "$(date -u '+%Y-%m-%dT%H:%M:%SZ')" \
+          | jq '[.Events[] | .CloudTrailEvent | fromjson]'
+    requires:
+      vars: [RUN] # RUN like 01JCDYYWMQGA3R2XQWH6ZM2HZN
+  get-userdata:
+    desc: list userdata for an ec2 intance -- supply the instance ID as CLI args
+    cmd: |
+      aws ec2 describe-instance-attribute --instance-id {{.CLI_ARGS}} --attribute userData --output text --query "UserData.Value" | base64 --decode
+  list-vpcs:
+    desc: list vpc's and their cidr block
+    cmd: |
+      aws ec2 describe-vpcs --query 'Vpcs[*].{VpcId:VpcId,Name:Tags[?Key==`Name`].Value|[0],CidrBlock:CidrBlock}' --output text

cfa-taskfiles/foo.sh

@@ -0,0 +1,30 @@
+#!/usr/bin/env bash
+
+set -o nounset
+set -o errexit
+#set -o xtrace
+
+fail() {
+  echo $1 >&2
+  exit 1
+}
+
+# yq eval                                                                                                                                                                  
+yaml_files=($(find . -name '*.yml' -print))
+if [ ${#yaml_files[@]} -gt 0 ]; then
+  # yq eval --exit-status evaluates files returns a non-zero exit status:
+  # override that exit status so this entire script doesn't exit prematurely
+  broken_files=()
+  for f in ${yaml_files[@]}; do
+    yq --exit-status 'tag == "!!map" or tag== "!!seq"' >/dev/null 2>&1 $f || broken_files+=($f)
+  done 
+  #echo "${broken_files[@]}"
+  #output=$(printf '%s\n' ${broken_files[@]})
+  #echo "$output"
+  if [ ${#broken_files[@]} -gt 0 ]; then
+    echo 'The following files appear to contain invalid yaml:' >&2
+  	printf '%s\n' ${broken_files[@]} >&2
+	  exit 1
+  fi
+fi
+

cfa-taskfiles/grafana.yml

@@ -0,0 +1,18 @@
+version: '3'
+
+tasks:
+ 
+  loki-port-forward:
+    desc: forward a local port to the loki gateway
+    cmds:
+      - cmd: |
+          printf 'run logcli like: env LOKI_ADDR=%s LOKI_ORG_ID=%s logcli <options>\n' $LOKI_ADDR $LOKI_ORG_ID
+        silent: true
+      - kubectl --namespace loki port-forward svc/loki-gateway --address $LOKI_LOCAL_HOST $LOKI_LOCAL_PORT:80
+        #  requires:
+        #      vars: [LOKI_LOCAL_HOST, LOKI_LOCAL_PORT, LOKI_ADDR, LOKI_ORG_ID]
+    env:
+      LOKI_LOCAL_HOST: 127.0.1.1
+      LOKI_LOCAL_PORT: 8000
+      LOKI_ADDR: http://{{.LOKI_LOCAL_HOST}}:{{.LOKI_LOCAL_PORT}}
+      LOKI_ORG_ID: ''

cfa-taskfiles/integrations.yml

@@ -0,0 +1,14 @@
+version: '3'
+
+tasks:
+  aws-cli:
+    desc: connect to the aws-cli pod running in the environment
+    cmds:
+      - cmd: kubectl exec -it aws-cli -- bash  
+  func-port-forward:
+    desc: forward a local port to the d20 functions listener
+    cmds:
+      - kubectl port-forward svc/d20-functions --address $D20_LOCALHOST $D20_FUNCTIONS_PORT:80
+    env:
+      D20_LOCALHOST: 127.0.2.1
+      D20_FUNCTIONS_PORT: 5080

cfa-taskfiles/k8s.yml

@@ -0,0 +1,32 @@
+version: '3'
+
+tasks:
+  shell-*:
+    desc: run a shell on pod with app matching label (try `task shell-d20-functions`)
+    cmds:
+      # - test -n "{{.POD}}" || (echo "No pod found, try task list-apps" && exit 1)
+      - kubectl exec -it {{.POD}} -- bash
+      # && kubectl exec -it {{.POD}} -- bash || echo "No pod found"
+      # - kubectl exec -it {{.POD}} -- bash
+    vars:
+      POD:
+        sh: kubectl get pods -l app={{index .MATCH 0}} -o name | head -1
+    preconditions:
+      - kubectl cluster-info
+      - test -n "{{.POD}}"
+  list-apps:
+    desc: list all apps in the environment
+    cmds:
+      - kubectl get pods -o json | jq -r '[ .items[] | select (.metadata.labels.app != null) | .metadata.labels.app ] | unique[]'
+      # - kubectl get pods -o jsonpath='{range .items[*]}{.metadata.labels.app}{"\n"}{end}' | sort | uniq
+    preconditions:
+      - kubectl cluster-info
+  argocd:
+    desc: port forward argocd ui
+    cmds:
+      - kubectl port-forward -n argocd svc/argocd-server --address 0.0.0.0 12345:80
+  alloy:
+    desc: port forward alloy ui
+    cmds:
+      - kubectl port-forward -n grafana svc/alloy --address 0.0.0.0 12345:12345
+

cfa-taskfiles/mobile.yml

@@ -0,0 +1,17 @@
+version: '3'
+
+tasks:
+  grafana:
+    desc: forward a local port to grafana
+    cmd: kubectl port-forward svc/kube-prometheus-stack-grafana --address 0.0.0.0 12345:80 --namespace monitoring
+  prometheus:
+    desc: forward port 12345 to the prometheus service
+    cmd: kubectl port-forward svc/kube-prometheus-stack-prometheus --address 0.0.0.0 12345:9090 --namespace monitoring
+  rabbitmq:
+    desc: forward a local port rabbitmq managment port
+    cmds:
+      - kubectl port-forward svc/rabbitmq --address 0.0.0.0 12345:15672
+  celery-logs:
+    desc: tail celery logs
+    cmd: kubectl logs -l pod-type=celery --tail 10 --timestamps -f --max-log-requests 16
+

cfa-taskfiles/pants.yml

@@ -0,0 +1,22 @@
+version: '3'
+
+env:
+  IMAGE: jswank/pants
+  TAG: latest
+
+tasks:
+  default:
+    cmds:
+      - task: build
+  build:
+    desc: build a new image
+    cmd: podman build -t ${IMAGE}:${TAG} {{.CLI_ARGS}} -f Dockerfile ctx
+  rebuild:
+    desc: rebuild a new image
+    cmds:
+      - podman rmi docker.io/library/python:3.10-bookworm
+      - task: build
+  run:
+    desc: run an ephemeral pants container, mounting the pants repo
+    cmd: podman run -ti -v $HOME/volumes/repos:/home/pants/repos --rm -w /home/pants/repos/ets-pants jswank/pants:latest
+

cfa-taskfiles/pyrenees.yml

@@ -0,0 +1,24 @@
+version: '3'
+
+vars:
+  SERVICE_ID: vpce-svc-08c9a7a8c06159a36  # the USE1 service ID for Pyrenees, override using -v SERVICE_ID=<>
+
+tasks:
+  list-tgws:
+    desc: list transits gateways with tag Name=ets-cloudops-pyrenees
+    cmd: |
+      aws ec2 describe-transit-gateways --filters Name=tag:Name,Values=ets-cloudops-pyrenees
+      # aws ec2 describe-transit-gateways --filters Name=tag:Name,Values=ets-cloudops-pyrenees | jq -r '.TransitGateways[] | {TransitGatewayId,State}'
+  list-services:
+    desc: list all Pyrenees endpoint services
+    cmd: |
+      aws ec2 describe-vpc-endpoint-services | jq -r '.ServiceDetails[] | select(.Tags[] | select(.Key == "Service" and .Value == "pyrenees-endpoint-service")) | .ServiceId'
+  list-principals:
+    desc: List principals for a regional endpoint service
+    cmd: |
+      aws ec2 describe-vpc-endpoint-service-permissions --service-id {{ .SERVICE_ID }} \
+        | jq -r '[.AllowedPrincipals[] | {Principal,Tags}]'
+  list-azs:
+    desc: list the availability zones for a regional endpoint service (works to validate that an account is an allowed principal)
+    cmd: |
+      aws ec2 describe-vpc-endpoint-services --service-ids {{ .SERVICE_ID }} | jq -r '.ServiceDetails.AvailabilityZones[]'

cfa-taskfiles/spacelift.yml

@@ -0,0 +1,37 @@
+version: '3'
+
+tasks:
+  login:
+    desc: login to spacelift
+    cmd: |
+      spacectl profile login
+  stacks:
+    desc: list stacks
+    cmd: |
+      spacectl stack list -o json  | jq -r '.[] | .id'
+  unconfirmed:
+    desc: list stacks with unconfirmed runs
+    cmd: |
+      spacectl stack list -o json  | jq '.[] | select(.state == "UNCONFIRMED") | {id,Blocker,trackedCommit}' 
+      spacectl stack list -o json  | jq -r '.[] | select(.state == "UNCONFIRMED") | "spacectl stack confirm --id \(.id) --run \(.Blocker.id)"'
+  runs:
+    desc: show recent runs for a stack (must supply stack id)
+    cmd: |
+     spacectl stack run list --id {{.STACK}}
+    requires:
+      vars: [STACK]
+  confirm:
+    desc: confirm a run (must supply stack id and run id as cli args)
+    cmd: |
+      spacectl stack confirm --id {{ .STACK }} --run {{ .RUN }}
+    requires:
+      vars: [STACK,RUN]
+
+  help:
+    desc: display some spacelift env help
+    silent: true
+    cmd: |
+      printf 'To run TF that uses the spacelift provider, do:\n'
+      printf '\tspacectl profile login\n'
+      printf '\texport SPACELIFT_API_TOKEN=$(spacectl profile export-token)\n'
+

misc/Taskfile.yml

@@ -0,0 +1,116 @@
+version: '3'
+
+vars:
+  GOOS:
+    sh: go env GOOS
+  GOARCH:
+    sh: go env GOARCH
+
+env:
+  DESTDIR: /usr/local/sbin
+
+tasks:
+
+  default:
+    cmds:
+      - task: build-all
+
+  setup:
+    desc: setup directories
+    internal: true 
+    cmds:
+      - mkdir -p bin
+    status:
+      - test -d bin
+
+  build-*:
+    desc: build a specific native binary
+    cmds:
+      - task: x-build-{{.BINARY}}
+      - cp bin/{{.BINARY}}_{{.GOOS}}_{{.GOARCH}} bin/{{.BINARY}}
+    vars:
+      BINARY: '{{index .MATCH 0}}'
+    sources:
+      - '**/*.go'
+    generates:
+      - bin/{{.BINARY}}
+
+  build-all:
+    desc: build all native binaries
+    cmds:
+      - for: { var: BINARIES }
+        task: build-{{.ITEM}}
+        vars:
+          BINARY: '{{.ITEM}}'
+    vars: 
+      BINARIES:
+        sh: ls cmd/
+
+  clean:
+    desc: clean up
+    cmd: rm -rf bin/*
+
+  install-*:
+    desc: install a binary
+    cmd: |
+      sudo install -m 0755 bin/{{.BINARY}} {{.DESTDIR}}/{{.BINARY}}
+    vars:
+      BINARY: '{{index .MATCH 0}}'
+
+  install-all:
+    desc: install all binaries
+    cmds:
+      - for: { var: BINARIES }
+        task: install-{{.ITEM}}
+    vars: 
+      BINARIES:
+        sh: ls cmd/
+
+  matrix-build-all:
+    desc: build all binaries for multiple os & architectures
+    cmds:
+      - for: { var: BINARIES }
+        task: matrix-build-{{.ITEM}}
+        vars:
+          BINARY: '{{.ITEM}}'
+    vars: 
+      BINARIES:
+        sh: ls cmd/
+
+  matrix-build-*:
+    desc: build a specific binary for multiple os & architectures
+    cmds:
+      - for:
+          matrix:
+            GOOS: [linux, darwin]
+            GOARCH: [amd64, arm64]
+        task: x-build-{{.BINARY}}
+        vars:
+          GOOS: '{{.ITEM.GOOS}}'
+          GOARCH: '{{.ITEM.GOARCH}}'
+      - task: x-build-{{.BINARY}}
+        vars:
+          GOOS: windows
+          GOARCH: amd64
+    vars:
+      BINARY: '{{index .MATCH 0}}'
+
+  x-build-*:
+    desc: build a binary for the OS / Arch specified by $GOOS and $GOARCH
+    cmd: |
+      GOOS={{.GOOS}} GOARCH={{.GOARCH}} go build -o bin/{{.BINARY}}_{{.GOOS}}_{{.GOARCH}} ./cmd/{{.BINARY}}
+    vars:
+      BINARY: '{{index .MATCH 0}}'
+    sources:
+      - '**/*.go'
+    generates:
+      - bin/{{.BINARY}}_{{.GOOS}}_{{.GOARCH}}
+    deps:
+      - setup
+
+  update-deps:
+    desc: update go dependencies
+    cmds:
+      - go get -u ./...
+      - go mod tidy
+