commit 8700494b9bb19c5c779d15637bcfc5e1be8c5485 Author: Jason Swank Date: Tue Dec 31 20:31:25 2024 -0500 initial commit diff --git a/.githooks/pre-commit b/.githooks/pre-commit new file mode 100755 index 0000000..d584653 --- /dev/null +++ b/.githooks/pre-commit @@ -0,0 +1,50 @@ +#!/usr/bin/env bash + +set -o errexit +set -o nounset + +fail() { + echo "$1" >&2 + exit 1 +} + +preflight() { + local cmd=$1 + local msg="%s not found: install it or use the --no-verify flag for the git commit to avoid the running this pre-commit hook.\n" + command -v $cmd >/dev/null 2>&1 || fail \ + "$(printf "$msg" $cmd)" +} + +# +# preflight checks +# +preflight terraform +preflight yq + +# terraform fmt -check +tf_files=($(git diff --cached --name-only --diff-filter=ACM -- '*.tf' '*.tfvars')) +if [ ${#tf_files[@]} -gt 0 ]; then + # terraform fmt -check returns a non-zero exit status and list of files to be + # formatted: override that exit status so this entire script doesn't exit prematurely + output=$(printf '%s\n' "${tf_files[@]}" | xargs terraform fmt -check 2>&1 || true) + if [ -n "$output" ]; then + printf 'The following files are incorrectly formated: run `terraform fmt` to fix them.\n' >&2 + fail $output + fi +fi + +# yq eval +yaml_files=($(git diff --cached --name-only --diff-filter=ACM -- '*.yml' '*.yaml')) +if [ ${#yaml_files[@]} -gt 0 ]; then + broken_files=() + for f in ${yaml_files[@]}; do + yq --exit-status 'tag == "!!map" or tag== "!!seq"' >/dev/null 2>&1 $f || broken_files+=($f) + done + if [ ${#broken_files[@]} -gt 0 ]; then + echo 'The following files appear to contain invalid yaml:' >&2 + fail "$(printf '%s\n' ${broken_files[@]})" +# echo 'The following files appear to contain invalid yaml:' >&2 +# printf '%s\n' ${broken_files[@]} >&2 +# exit 1 + fi +fi diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..4c49bd7 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +.env diff --git a/README.md b/README.md new file mode 100644 index 0000000..c29da10 --- /dev/null +++ b/README.md @@ -0,0 +1,9 @@ +# taskfiles + +Here are some [taskfiles](https://taskfile.dev/) I use. + +## Hooks +Git hooks for this repo are provided. To use them, configure the `hooksPath` for the repo: +``` +$ git config --local core.hooksPath .githooks +``` diff --git a/Taskfile.yml b/Taskfile.yml new file mode 100644 index 0000000..9d36240 --- /dev/null +++ b/Taskfile.yml @@ -0,0 +1,11 @@ +version: '3' + +includes: + aws: ./cfa-taskfiles/aws.yml + int: ./cfa-taskfiles/integrations.yml + mobile: ./cfa-taskfiles/mobile.yml + k8s: ./cfa-taskfiles/k8s.yml + grafana: ./cfa-taskfiles/grafana.yml + pyrenees: ./cfa-taskfiles/pyrenees.yml + spacelift: ./cfa-taskfiles/spacelift.yml + diff --git a/bootstrap/Taskfile-alpine.yml b/bootstrap/Taskfile-alpine.yml new file mode 100644 index 0000000..ea6dc08 --- /dev/null +++ b/bootstrap/Taskfile-alpine.yml @@ -0,0 +1,29 @@ +version: '3' + +tasks: + default: + desc: initialize system + cmds: + - task: packages + - task: dotfiles + - task: misc + + packages: + desc: install packages + cmd: | + sudo /sbin/apk -U add \ + tmux zsh zsh-vcs git neovim py3-pynvim stow fzf \ + > /dev/null + + dotfiles: + desc: install my dotfiles + dir: $HOME + cmds: + - test -d dotfiles || git clone https://git.scalene.net/jswank/dotfiles.git > /dev/null + - cd dotfiles && ./install.sh > /dev/null + + misc: + cmds: + - chsh -s /bin/zsh + - mkdir -p $HOME/.local/bin + - curl -Ls https://github.com/marwanhawari/stew/releases/download/v0.4.0/stew-v0.4.0-linux-amd64.tar.gz | tar -xz stew -C $HOME/.local/bin diff --git a/bootstrap/Taskfile-ubuntu.yml b/bootstrap/Taskfile-ubuntu.yml new file mode 100644 index 0000000..78f627f --- /dev/null +++ b/bootstrap/Taskfile-ubuntu.yml @@ -0,0 +1,28 @@ +version: '3' + +tasks: + default: + desc: initialize system + cmds: + - task: packages + - task: dotfiles + - task: misc + + packages: + desc: install packages + cmds: + - sudo /usr/bin/apt update --yes + - sudo /usr/bin/apt install --yes tmux zsh git neovim stow fzf + + dotfiles: + desc: install my dotfiles + dir: $HOME + cmds: + - test -d dotfiles || git clone https://git.scalene.net/jswank/dotfiles.git > /dev/null + - cd dotfiles && ./install.sh > /dev/null + + misc: + cmds: + - sudo chsh -s $(which zsh) $(whoami) + - mkdir -p $HOME/.local/bin + - curl -Ls https://github.com/marwanhawari/stew/releases/download/v0.4.0/stew-v0.4.0-linux-amd64.tar.gz | tar -xz stew -C $HOME/.local/bin diff --git a/bootstrap/Taskfile.yml b/bootstrap/Taskfile.yml new file mode 100644 index 0000000..77cce86 --- /dev/null +++ b/bootstrap/Taskfile.yml @@ -0,0 +1,26 @@ +version: '3' + +tasks: + default: + desc: initialize system + cmds: + - task: packages + - task: dotfiles + - task: misc + + packages: + desc: install packages + cmds: + - sudo /usr/bin/apt update --yes + - sudo /usr/bin/apt install --yes tmux zsh git neovim stow fzf + + dotfiles: + desc: install my dotfiles + dir: $HOME + cmds: + - test -d dotfiles || git clone https://git.scalene.net/jswank/dotfiles.git > /dev/null + - cd dotfiles && ./install.sh > /dev/null + + misc: + cmds: + - sudo chsh -s $(which zsh) $(whoami) diff --git a/bootstrap/bootstrap-ubuntu b/bootstrap/bootstrap-ubuntu new file mode 100644 index 0000000..a0f2813 --- /dev/null +++ b/bootstrap/bootstrap-ubuntu @@ -0,0 +1,14 @@ +#!/usr/bin/env bash + +set -o nounset +set -o errexit + +cd ~/ + +mkdir -p .local/bin + +if [[ ! -x ~/.local/bin/task ]]; then + sh -c "$(curl --silent --location https://taskfile.dev/install.sh)" -- -d -b ~/.local/bin +fi + +~/.local/bin/task diff --git a/cfa-taskfiles/aws.yml b/cfa-taskfiles/aws.yml new file mode 100644 index 0000000..e45496e --- /dev/null +++ b/cfa-taskfiles/aws.yml @@ -0,0 +1,99 @@ +version: '3' + +tasks: + subshell: + desc: invoke a subshell w/ correct AWS variables set + cmds: + - cmd: | + eval `okta-aws-cli web`; AWS_ENVIRONMENT="$(aws iam list-account-aliases | jq -r '.AccountAliases[0]')" zsh + - cmd: rm -f {{ .USER_WORKING_DIR }}/.session-env + logout: + desc: Invalidate the OKTA credential + cmd: rm -f ${HOME}/.okta/awscli-access-token.json + preconditions: + - test -f ${HOME}/.okta/awscli-access-token.json + list-secrets: + desc: List all secrets + cmd: | + aws secretsmanager list-secrets | jq '[.SecretList[] | {"name" :.Name, "arn": .ARN, "desc": .Description}]' + get-secret: + desc: Get a secret - supply the name the secret as arg1 + cmd: | + aws secretsmanager get-secret-value --secret-id {{.CLI_ARGS}} --query SecretString --output text + list-load-balancers: + desc: List all load balancers + cmd: aws elbv2 describe-load-balancers | jq -r '.LoadBalancers[].LoadBalancerArn' + list-listeners: + desc: List all listners for a load balancer - supply the load balancer ARN as arg1 + cmd: | + aws elbv2 describe-listeners --load-balancer-arn {{.CLI_ARGS}} | jq -r '.Listeners[].ListenerArn' + get-rules: + desc: get rules - supply listener ARN as arg1 + cmd: | + aws elbv2 describe-rules --listener-arn {{.CLI_ARGS}} | gron |grep HostHeaderConfig.Values |grep -v '];' +# get-policy: +# desc: return the latest version of the specified policy +# cmd: | +# vars: +# POLICY_NAME: +# POLICY_ID: + list-clusters: + desc: list EKS clusters + cmd: aws eks list-clusters | jq -r '.clusters[]' + kubeconfig: + desc: update kubeconfig for the given name + cmd: aws eks update-kubeconfig --name {{.CLI_ARGS}} + infer-kubeconfig: + desc: infer kubeconfig based on ETS SRE conventions + # cmd: aws eks update-kubeconfig --name {{.CLI_ARGS}} + cmd: aws eks update-kubeconfig --name {{.CLUSTER_NAME}} + vars: + CLUSTER_NAME: + sh: aws eks list-clusters | jq -r '.clusters[0]' + connect-alloy-pod: + desc: run a shell on an alloy pod + cmd: kubectl exec -it alloy-6qflx --namespace grafana -- /bin/bash + forward-alloy-pod: + desc: port forward an alloy pod + cmd: kubectl port-forward alloy-6qflx --address 0.0.0.0 12345:12345 --namespace grafana + infer-alb-hosts: + desc: infer hostnames supported by an ALB based on ETS SRE conventions + silent: true + cmd: | + #!/bin/sh + set -o errexit + set -o pipefail + alb_arn=$(aws elbv2 describe-load-balancers | jq -r '.LoadBalancers[].LoadBalancerArn' | grep awsingress) + listener_arn=$(aws elbv2 describe-listeners --load-balancer-arn $alb_arn | jq -r '.Listeners[].ListenerArn') + aws elbv2 describe-rules --listener-arn "$listener_arn" | gron | grep 'HostHeaderConfig.Values\[' | perl -nE 'say $1 if m/"(.+)"/' + get-spacelift-runs: + desc: return a list of all spacelift runs in the last 4 hours + cmd: | + aws cloudtrail lookup-events \ + --lookup-attributes AttributeKey=EventName,AttributeValue=AssumeRoleWithWebIdentity \ + --start-time "$(date -u -d '-240 minutes' '+%Y-%m-%dT%H:%M:%SZ')" \ + --end-time "$(date -u '+%Y-%m-%dT%H:%M:%SZ')" \ + | jq -r '.Events[] + | select(.Resources[].ResourceName + | endswith(":role/spacelift")) + | .Resources[] + | select(.ResourceType == "AWS::STS::AssumedRole" and (.ResourceName | type == "string" and startswith("spacelift-run"))) + | .ResourceName' + get-spacelift-run: + desc: get all events for a specific spacelift RUN id which occured in the last 24 hours + cmd: | + aws cloudtrail lookup-events \ + --lookup-attributes AttributeKey=Username,AttributeValue=spacelift-run-{{ .RUN }} \ + --start-time "$(date -u -d '-24 hours' '+%Y-%m-%dT%H:%M:%SZ')" \ + --end-time "$(date -u '+%Y-%m-%dT%H:%M:%SZ')" \ + | jq '[.Events[] | .CloudTrailEvent | fromjson]' + requires: + vars: [RUN] # RUN like 01JCDYYWMQGA3R2XQWH6ZM2HZN + get-userdata: + desc: list userdata for an ec2 intance -- supply the instance ID as CLI args + cmd: | + aws ec2 describe-instance-attribute --instance-id {{.CLI_ARGS}} --attribute userData --output text --query "UserData.Value" | base64 --decode + list-vpcs: + desc: list vpc's and their cidr block + cmd: | + aws ec2 describe-vpcs --query 'Vpcs[*].{VpcId:VpcId,Name:Tags[?Key==`Name`].Value|[0],CidrBlock:CidrBlock}' --output text diff --git a/cfa-taskfiles/foo.sh b/cfa-taskfiles/foo.sh new file mode 100755 index 0000000..b281a6b --- /dev/null +++ b/cfa-taskfiles/foo.sh @@ -0,0 +1,30 @@ +#!/usr/bin/env bash + +set -o nounset +set -o errexit +#set -o xtrace + +fail() { + echo $1 >&2 + exit 1 +} + +# yq eval +yaml_files=($(find . -name '*.yml' -print)) +if [ ${#yaml_files[@]} -gt 0 ]; then + # yq eval --exit-status evaluates files returns a non-zero exit status: + # override that exit status so this entire script doesn't exit prematurely + broken_files=() + for f in ${yaml_files[@]}; do + yq --exit-status 'tag == "!!map" or tag== "!!seq"' >/dev/null 2>&1 $f || broken_files+=($f) + done + #echo "${broken_files[@]}" + #output=$(printf '%s\n' ${broken_files[@]}) + #echo "$output" + if [ ${#broken_files[@]} -gt 0 ]; then + echo 'The following files appear to contain invalid yaml:' >&2 + printf '%s\n' ${broken_files[@]} >&2 + exit 1 + fi +fi + diff --git a/cfa-taskfiles/grafana.yml b/cfa-taskfiles/grafana.yml new file mode 100644 index 0000000..142b466 --- /dev/null +++ b/cfa-taskfiles/grafana.yml @@ -0,0 +1,18 @@ +version: '3' + +tasks: + + loki-port-forward: + desc: forward a local port to the loki gateway + cmds: + - cmd: | + printf 'run logcli like: env LOKI_ADDR=%s LOKI_ORG_ID=%s logcli \n' $LOKI_ADDR $LOKI_ORG_ID + silent: true + - kubectl --namespace loki port-forward svc/loki-gateway --address $LOKI_LOCAL_HOST $LOKI_LOCAL_PORT:80 + # requires: + # vars: [LOKI_LOCAL_HOST, LOKI_LOCAL_PORT, LOKI_ADDR, LOKI_ORG_ID] + env: + LOKI_LOCAL_HOST: 127.0.1.1 + LOKI_LOCAL_PORT: 8000 + LOKI_ADDR: http://{{.LOKI_LOCAL_HOST}}:{{.LOKI_LOCAL_PORT}} + LOKI_ORG_ID: '' diff --git a/cfa-taskfiles/integrations.yml b/cfa-taskfiles/integrations.yml new file mode 100644 index 0000000..ff9ba1c --- /dev/null +++ b/cfa-taskfiles/integrations.yml @@ -0,0 +1,14 @@ +version: '3' + +tasks: + aws-cli: + desc: connect to the aws-cli pod running in the environment + cmds: + - cmd: kubectl exec -it aws-cli -- bash + func-port-forward: + desc: forward a local port to the d20 functions listener + cmds: + - kubectl port-forward svc/d20-functions --address $D20_LOCALHOST $D20_FUNCTIONS_PORT:80 + env: + D20_LOCALHOST: 127.0.2.1 + D20_FUNCTIONS_PORT: 5080 diff --git a/cfa-taskfiles/k8s.yml b/cfa-taskfiles/k8s.yml new file mode 100644 index 0000000..ae92020 --- /dev/null +++ b/cfa-taskfiles/k8s.yml @@ -0,0 +1,32 @@ +version: '3' + +tasks: + shell-*: + desc: run a shell on pod with app matching label (try `task shell-d20-functions`) + cmds: + # - test -n "{{.POD}}" || (echo "No pod found, try task list-apps" && exit 1) + - kubectl exec -it {{.POD}} -- bash + # && kubectl exec -it {{.POD}} -- bash || echo "No pod found" + # - kubectl exec -it {{.POD}} -- bash + vars: + POD: + sh: kubectl get pods -l app={{index .MATCH 0}} -o name | head -1 + preconditions: + - kubectl cluster-info + - test -n "{{.POD}}" + list-apps: + desc: list all apps in the environment + cmds: + - kubectl get pods -o json | jq -r '[ .items[] | select (.metadata.labels.app != null) | .metadata.labels.app ] | unique[]' + # - kubectl get pods -o jsonpath='{range .items[*]}{.metadata.labels.app}{"\n"}{end}' | sort | uniq + preconditions: + - kubectl cluster-info + argocd: + desc: port forward argocd ui + cmds: + - kubectl port-forward -n argocd svc/argocd-server --address 0.0.0.0 12345:80 + alloy: + desc: port forward alloy ui + cmds: + - kubectl port-forward -n grafana svc/alloy --address 0.0.0.0 12345:12345 + diff --git a/cfa-taskfiles/mobile.yml b/cfa-taskfiles/mobile.yml new file mode 100644 index 0000000..8c89dfc --- /dev/null +++ b/cfa-taskfiles/mobile.yml @@ -0,0 +1,17 @@ +version: '3' + +tasks: + grafana: + desc: forward a local port to grafana + cmd: kubectl port-forward svc/kube-prometheus-stack-grafana --address 0.0.0.0 12345:80 --namespace monitoring + prometheus: + desc: forward port 12345 to the prometheus service + cmd: kubectl port-forward svc/kube-prometheus-stack-prometheus --address 0.0.0.0 12345:9090 --namespace monitoring + rabbitmq: + desc: forward a local port rabbitmq managment port + cmds: + - kubectl port-forward svc/rabbitmq --address 0.0.0.0 12345:15672 + celery-logs: + desc: tail celery logs + cmd: kubectl logs -l pod-type=celery --tail 10 --timestamps -f --max-log-requests 16 + diff --git a/cfa-taskfiles/pants.yml b/cfa-taskfiles/pants.yml new file mode 100644 index 0000000..f1730b0 --- /dev/null +++ b/cfa-taskfiles/pants.yml @@ -0,0 +1,22 @@ +version: '3' + +env: + IMAGE: jswank/pants + TAG: latest + +tasks: + default: + cmds: + - task: build + build: + desc: build a new image + cmd: podman build -t ${IMAGE}:${TAG} {{.CLI_ARGS}} -f Dockerfile ctx + rebuild: + desc: rebuild a new image + cmds: + - podman rmi docker.io/library/python:3.10-bookworm + - task: build + run: + desc: run an ephemeral pants container, mounting the pants repo + cmd: podman run -ti -v $HOME/volumes/repos:/home/pants/repos --rm -w /home/pants/repos/ets-pants jswank/pants:latest + diff --git a/cfa-taskfiles/pyrenees.yml b/cfa-taskfiles/pyrenees.yml new file mode 100644 index 0000000..9f585dd --- /dev/null +++ b/cfa-taskfiles/pyrenees.yml @@ -0,0 +1,24 @@ +version: '3' + +vars: + SERVICE_ID: vpce-svc-08c9a7a8c06159a36 # the USE1 service ID for Pyrenees, override using -v SERVICE_ID=<> + +tasks: + list-tgws: + desc: list transits gateways with tag Name=ets-cloudops-pyrenees + cmd: | + aws ec2 describe-transit-gateways --filters Name=tag:Name,Values=ets-cloudops-pyrenees + # aws ec2 describe-transit-gateways --filters Name=tag:Name,Values=ets-cloudops-pyrenees | jq -r '.TransitGateways[] | {TransitGatewayId,State}' + list-services: + desc: list all Pyrenees endpoint services + cmd: | + aws ec2 describe-vpc-endpoint-services | jq -r '.ServiceDetails[] | select(.Tags[] | select(.Key == "Service" and .Value == "pyrenees-endpoint-service")) | .ServiceId' + list-principals: + desc: List principals for a regional endpoint service + cmd: | + aws ec2 describe-vpc-endpoint-service-permissions --service-id {{ .SERVICE_ID }} \ + | jq -r '[.AllowedPrincipals[] | {Principal,Tags}]' + list-azs: + desc: list the availability zones for a regional endpoint service (works to validate that an account is an allowed principal) + cmd: | + aws ec2 describe-vpc-endpoint-services --service-ids {{ .SERVICE_ID }} | jq -r '.ServiceDetails.AvailabilityZones[]' diff --git a/cfa-taskfiles/spacelift.yml b/cfa-taskfiles/spacelift.yml new file mode 100644 index 0000000..7043144 --- /dev/null +++ b/cfa-taskfiles/spacelift.yml @@ -0,0 +1,37 @@ +version: '3' + +tasks: + login: + desc: login to spacelift + cmd: | + spacectl profile login + stacks: + desc: list stacks + cmd: | + spacectl stack list -o json | jq -r '.[] | .id' + unconfirmed: + desc: list stacks with unconfirmed runs + cmd: | + spacectl stack list -o json | jq '.[] | select(.state == "UNCONFIRMED") | {id,Blocker,trackedCommit}' + spacectl stack list -o json | jq -r '.[] | select(.state == "UNCONFIRMED") | "spacectl stack confirm --id \(.id) --run \(.Blocker.id)"' + runs: + desc: show recent runs for a stack (must supply stack id) + cmd: | + spacectl stack run list --id {{.STACK}} + requires: + vars: [STACK] + confirm: + desc: confirm a run (must supply stack id and run id as cli args) + cmd: | + spacectl stack confirm --id {{ .STACK }} --run {{ .RUN }} + requires: + vars: [STACK,RUN] + + help: + desc: display some spacelift env help + silent: true + cmd: | + printf 'To run TF that uses the spacelift provider, do:\n' + printf '\tspacectl profile login\n' + printf '\texport SPACELIFT_API_TOKEN=$(spacectl profile export-token)\n' + diff --git a/misc/Taskfile.yml b/misc/Taskfile.yml new file mode 100644 index 0000000..cb4096d --- /dev/null +++ b/misc/Taskfile.yml @@ -0,0 +1,116 @@ +version: '3' + +vars: + GOOS: + sh: go env GOOS + GOARCH: + sh: go env GOARCH + +env: + DESTDIR: /usr/local/sbin + +tasks: + + default: + cmds: + - task: build-all + + setup: + desc: setup directories + internal: true + cmds: + - mkdir -p bin + status: + - test -d bin + + build-*: + desc: build a specific native binary + cmds: + - task: x-build-{{.BINARY}} + - cp bin/{{.BINARY}}_{{.GOOS}}_{{.GOARCH}} bin/{{.BINARY}} + vars: + BINARY: '{{index .MATCH 0}}' + sources: + - '**/*.go' + generates: + - bin/{{.BINARY}} + + build-all: + desc: build all native binaries + cmds: + - for: { var: BINARIES } + task: build-{{.ITEM}} + vars: + BINARY: '{{.ITEM}}' + vars: + BINARIES: + sh: ls cmd/ + + clean: + desc: clean up + cmd: rm -rf bin/* + + install-*: + desc: install a binary + cmd: | + sudo install -m 0755 bin/{{.BINARY}} {{.DESTDIR}}/{{.BINARY}} + vars: + BINARY: '{{index .MATCH 0}}' + + install-all: + desc: install all binaries + cmds: + - for: { var: BINARIES } + task: install-{{.ITEM}} + vars: + BINARIES: + sh: ls cmd/ + + matrix-build-all: + desc: build all binaries for multiple os & architectures + cmds: + - for: { var: BINARIES } + task: matrix-build-{{.ITEM}} + vars: + BINARY: '{{.ITEM}}' + vars: + BINARIES: + sh: ls cmd/ + + matrix-build-*: + desc: build a specific binary for multiple os & architectures + cmds: + - for: + matrix: + GOOS: [linux, darwin] + GOARCH: [amd64, arm64] + task: x-build-{{.BINARY}} + vars: + GOOS: '{{.ITEM.GOOS}}' + GOARCH: '{{.ITEM.GOARCH}}' + - task: x-build-{{.BINARY}} + vars: + GOOS: windows + GOARCH: amd64 + vars: + BINARY: '{{index .MATCH 0}}' + + x-build-*: + desc: build a binary for the OS / Arch specified by $GOOS and $GOARCH + cmd: | + GOOS={{.GOOS}} GOARCH={{.GOARCH}} go build -o bin/{{.BINARY}}_{{.GOOS}}_{{.GOARCH}} ./cmd/{{.BINARY}} + vars: + BINARY: '{{index .MATCH 0}}' + sources: + - '**/*.go' + generates: + - bin/{{.BINARY}}_{{.GOOS}}_{{.GOARCH}} + deps: + - setup + + update-deps: + desc: update go dependencies + cmds: + - go get -u ./... + - go mod tidy +