tahoe-lafs/release.py

#! /usr/bin/env python
# Tahoe-LAFS -- secure, distributed storage grid
#
# Copyright © 2006-2012 The Tahoe-LAFS Software Foundation
#
# This file is part of Tahoe-LAFS.
#
# See the docs/about.rst file for licensing information.
import os
import shutil
import re
import sys
import glob
import subprocess
import argparse
import datetime
import pathlib


class bcolors:
    HEADER = "\033[95m"
    OKBLUE = "\033[94m"
    OKCYAN = "\033[96m"
    OKGREEN = "\033[92m"
    WARNING = "\033[93m"
    FAIL = "\033[91m"
    ENDC = "\033[0m"
    BOLD = "\033[1m"
    UNDERLINE = "\033[4m"


parser = argparse.ArgumentParser()
parser.add_argument(
    "--clean", action="store_true", help="Cancel existing release process, clean files"
)
parser.add_argument(
    "--ignore-deps", action="store_true", help="Ignore dependency checks"
)
parser.add_argument("--fin", action="store_true", help="Finish release proccess")
parser.add_argument("--sign", type=str, help="Signing key")
parser.add_argument("--ticket", type=int, help="Ticket number", required=True)
parser.add_argument(
    "--repo",
    type=str,
    help="Destination repo (fork), example: git@github.com:meejah/tahoe-lafs.git",
)
parser.add_argument("--retry", action="store_true", help="Retry release")
parser.add_argument("--tag", type=str, help="Release tag", required=True)
args = parser.parse_args()


TAG = args.tag
TICKET = args.ticket
TODAY = datetime.date.today()
BRANCH = "{ticket}.release-{tag}".format(
    ticket=TICKET, tag=TAG
)  # looks like XXXX.release-1.18.0
RELEASE_TITLE = "Release {tag} ({date})".format(tag=TAG, date=TODAY)
RELEASE_FOLDER = pathlib.Path("../tahoe-release-{0}".format(TAG))
RELEASE_PROGRESS = RELEASE_FOLDER.joinpath(".release-progress")
CONTINUE_INSTRUCTION = (
    bcolors.BOLD
    + "Instruction : run {path}/venv/bin/python release.py --ignore-deps --tag {tag} --ticket {ticket} --sign {sign} --repo {repo} --fin".format(
        path=RELEASE_FOLDER,
        tag=TAG,
        ticket=TICKET,
        sign=args.sign if args.sign else "YOUR_SIGNING_KEY_HERE",
        repo=args.repo if args.repo else "origin",
    )
    + bcolors.ENDC
)


def clean():
    """
    Remove all files and folders generated by release process
    """
    shutil.rmtree(RELEASE_FOLDER, ignore_errors=True)
    shutil.rmtree(RELEASE_PROGRESS, ignore_errors=True)


def check_dependencies():
    """
    Check requirements before starting release
    """
    try:
        import wheel
    except ModuleNotFoundError:
        print(
            f"{bcolors.WARNING}Warning: wheel is not installed. Install via pip!{bcolors.ENDC}"
        )
        sys.exit(1)


def step_complete(step):
    """
    Check if step completed successfully
    """
    return args.retry and os.path.isfile(RELEASE_PROGRESS.joinpath(step))


def mkfile(path):
    """
    Create new file in path
    """
    with open(path, "w") as f:
        pass


def record_step(step):
    """
    Record successful completion of step
    """
    mkfile(RELEASE_PROGRESS.joinpath(step))


def start_release():
    """
    Start release: make fresh clone, release env, release branch, generate news
    """
    if step_complete("clone_complete"):
        print(f"{bcolors.OKCYAN}Skipping clone step...{bcolors.ENDC}")
    else:
        try:
            subprocess.run(
                [
                    "git",
                    "clone",
                    "https://github.com/tahoe-lafs/tahoe-lafs.git",
                    RELEASE_FOLDER,
                ],
                check=True,
            )
            record_step("clone_complete")
        except Exception as e:
            print(f"{bcolors.FAIL}INFO: Failed to clone! Exiting :(...{bcolors.ENDC}")
            print(f"{bcolors.FAIL} {e} {bcolors.ENDC}")
            sys.exit(1)
    os.chdir(RELEASE_FOLDER)
    if step_complete("install_deps_on_venv_complete"):
        print(
            f"{bcolors.OKCYAN}Skipping venv setup and dependency installation...{bcolors.ENDC}"
        )
    else:
        try:
            subprocess.run(["python", "-m", "venv", "venv"], check=True)
            subprocess.run(
                ["./venv/bin/pip", "install", "--editable", ".[test]"], check=True
            )
            record_step("install_deps_on_venv_complete")
        except Exception as e:
            print(
                f"{bcolors.FAIL}INFO: Failed to install virtualenv and and dependencies clone! :(...{bcolors.ENDC}"
            )
            print(f"{bcolors.FAIL} {e} {bcolors.ENDC}")
            sys.exit(1)
    if step_complete("branch_complete"):
        print(f"{bcolors.OKCYAN}Skipping create release branch...{bcolors.ENDC}")
    else:
        try:
            subprocess.run(["git", "branch", BRANCH], check=True)
            record_step("branch_complete")
        except Exception as e:
            print(
                f"{bcolors.FAIL}INFO: Failed to create release branch! :(...{bcolors.ENDC}"
            )
            print(f"{bcolors.FAIL} {e} {bcolors.ENDC}")
            sys.exit(1)
    subprocess.run(["git", "checkout", BRANCH])
    if step_complete("tox_news_complete"):
        print(f"{bcolors.OKCYAN}Skipping news generation...{bcolors.ENDC}")
    else:
        try:
            subprocess.run(["./venv/bin/tox", "-e", "news"], check=True)
            record_step("tox_news_complete")
        except Exception as e:
            print(f"{bcolors.FAIL}INFO: Failed to generate news! :(...{bcolors.ENDC}")
            print(f"{bcolors.FAIL} {e} {bcolors.ENDC}")
            sys.exit(1)
    if step_complete("newsfragment_complete"):
        print(f"{bcolors.OKCYAN}Skipping add news fragment...{bcolors.ENDC}")
    else:
        try:
            mkfile("newsfragments/{ticket}.minor".format(ticket=TICKET))
            record_step("newsfragment_complete")
        except Exception as e:
            print(
                f"{bcolors.FAIL}INFO: Failed to add newsfragment file! :(...{bcolors.ENDC}"
            )
            print(f"{bcolors.FAIL} {e} {bcolors.ENDC}")
            sys.exit(1)
    if step_complete("commit_newsfragment_complete"):
        print(f"{bcolors.OKCYAN}Skipping commit fragment...{bcolors.ENDC}")
    else:
        try:
            subprocess.run(
                ["git", "add", "newsfragments/{ticket}.minor".format(ticket=TICKET)],
                check=True,
            )
            subprocess.run(
                ["git", "commit", "-s", "-m", "tahoe-lafs-{tag} news".format(tag=TAG)],
                check=True,
            )
            record_step("commit_newsfragment_complete")
        except Exception as e:
            print(
                f"{bcolors.FAIL}INFO: Failed to commit newsfragment! :(...{bcolors.ENDC}"
            )
            print(f"{bcolors.FAIL} {e} {bcolors.ENDC}")
            sys.exit(1)
    updated_content = None
    LINE = "=" * len(RELEASE_TITLE)
    with open("NEWS.rst", "r") as f:
        content = f.read()
        updated_content = re.sub(
            r"\.\.\stowncrier start line\n(Release\s(\d+\.\d+\.\d)(\.)post\d+\s\(\d{4}-\d{02}-\d{02}\))+(\n\'+)",
            RELEASE_TITLE + "\n" + LINE,
            content,
        )
    with open("updated_news.rst", "w") as f:
        f.write(updated_content)
    shutil.move("updated_news.rst", "NEWS.rst")
    print(f"{bcolors.OKGREEN}First set of release steps completed! :).{bcolors.ENDC}")
    print(
        f"{bcolors.OKBLUE}Instruction: Move (cd) into the  release folder : {RELEASE_FOLDER}.{bcolors.ENDC}"
    )
    print(
        f"{bcolors.OKBLUE}Instruction: Please review {RELEASE_FOLDER.joinpath('NEWS.rst')} {bcolors.ENDC}"
    )
    print(
        f"{bcolors.OKBLUE}Instruction: Update 'docs/known_issues.rst' (if necessary){bcolors.ENDC}"
    )
    print(
        f"{bcolors.OKBLUE}Instruction: If any, commit changes to {RELEASE_FOLDER.joinpath('NEWS.rst')} and 'docs/known_issues.rst' {bcolors.ENDC}"
    )
    if not args.repo:
        print(
            f"{bcolors.OKBLUE}Instruction: If your intention IS NOT to push to \"origin (git@github.com:tahoe-lafs/tahoe-lafs.git)\", please add and set the '--repo' (example : --repo git@github.com:meejah/tahoe-lafs.git) flag to release complete command before executing"
        )
    if not args.sign:
        print(
            f"{bcolors.OKBLUE}Instruction: Modify the continue instruction and include your signing key via the '--sign' flag (example : --sign XXXXXXXXSAMPLEKEYXXXXXXXXX)"
        )
    if args.repo and ("http:" in args.repo or "https:" in args.repo):
        print(
            f"{bcolors.OKBLUE}Instruction: You would have to type in your github username/password, keep it near."
        )
    print(
        f"{bcolors.OKBLUE}Instruction: Make sure your gpg passphrase is ready for the next step."
    )
    print(CONTINUE_INSTRUCTION)


def complete_release():
    """
    Finalize release: push release branch, make and sign artifacts
    """
    os.chdir(RELEASE_FOLDER)
    if step_complete("push_branch"):
        print(f"{bcolors.OKCYAN}Skipping push release branch...{bcolors.ENDC}")
    else:
        try:
            REPO = args.repo if args.repo else "origin"
            subprocess.run(["git", "push", REPO, BRANCH], check=True)
            record_step("push_branch")
        except Exception as e:
            print(
                f"{bcolors.FAIL}INFO: Failed to push release branch :(...{bcolors.ENDC}"
            )
            print(f"{bcolors.FAIL} {e} {bcolors.ENDC}")
            sys.exit(1)
    if step_complete("sign_tag"):
        print(f"{bcolors.OKCYAN}Skipping sign release tag...{bcolors.ENDC}")
    else:
        try:
            SIGNING_KEY = args.sign
            subprocess.run(
                [
                    "git",
                    "tag",
                    "-s",
                    "-u",
                    SIGNING_KEY,
                    "-m",
                    RELEASE_TITLE.lower(),
                    TAG,
                ],
                check=True,
            )
            record_step("sign_tag")
        except Exception as e:
            print(
                f"{bcolors.FAIL}INFO: Failed to sign release tag, exiting :(...{bcolors.ENDC}"
            )
            print(f"{bcolors.FAIL} {e} {bcolors.ENDC}")
            sys.exit(1)
    if step_complete("build_code"):
        print(f"{bcolors.OKCYAN}Skipping local code build...{bcolors.ENDC}")
    else:
        try:
            subprocess.run(
                ["./venv/bin/tox", "-e", "py37,codechecks,docs,integration"], check=True
            )
            subprocess.run(
                ["./venv/bin/tox", "-e", "deprecations,upcoming-deprecations"],
                check=True,
            )
            record_step("build_code")
        except Exception as e:
            print(
                f"{bcolors.FAIL}INFO: Failed to build code locally :(...{bcolors.ENDC}"
            )
            print(f"{bcolors.FAIL} {e} {bcolors.ENDC}")
            sys.exit(1)
    if step_complete("build_tarballs"):
        print(
            f"{bcolors.OKCYAN}Skipping build tarballs, using old tarballs...{bcolors.ENDC}"
        )
    else:
        try:
            subprocess.run(["./venv/bin/tox", "-e", "tarballs"], check=True)
            record_step("build_tarballs")
        except Exception as e:
            print(
                f"{bcolors.FAIL}INFO: Failed build release artifacts :(...{bcolors.ENDC}"
            )
            print(f"{bcolors.FAIL} {e} {bcolors.ENDC}")
            sys.exit(1)
    if step_complete("sign_tarballs"):
        print(
            f"{bcolors.OKCYAN}Skipping sign tarballs, using old tarballs...{bcolors.ENDC}"
        )
    else:
        try:
            paths = [
                pathlib.Path(p)
                for p in glob.glob(str(RELEASE_FOLDER.joinpath(f"dist/*")))
            ]
            for path in paths:
                subprocess.run(
                    [
                        "gpg",
                        "--pinentry=loopback",
                        "--armor",
                        "--detach-sign",
                        path,
                    ],
                    check=True,
                )
            record_step("sign_tarballs")
        except Exception as e:
            print(f"{bcolors.FAIL}INFO: Failed to sign tarballs :(...{bcolors.ENDC}")
            print(f"{bcolors.FAIL} {e} {bcolors.ENDC}")
            sys.exit(1)
    print(f"{bcolors.OKGREEN}Cleaning working files...{bcolors.ENDC}")
    clean()
    print(f"{bcolors.OKGREEN}Release complete...{bcolors.ENDC}")


if args.clean and not args.retry:
    print(f"{bcolors.OKCYAN}Start cleaning...{bcolors.ENDC}")
    clean()
    print(f"{bcolors.OKGREEN}Cleaning complete...{bcolors.ENDC}")
if args.retry:
    print(
        f"{bcolors.OKCYAN}Picking up from last try in {RELEASE_FOLDER}...{bcolors.ENDC}"
    )
if args.fin:
    if args.sign is None:
        print("Signing key required to complete release process")
        sys.exit(1)
    complete_release()
    print(f"{bcolors.OKGREEN}INFO: Release procedure complete! :)...{bcolors.ENDC}")
    sys.exit(0)
if not args.ignore_deps:
    check_dependencies()

if not os.path.exists(RELEASE_PROGRESS):
    os.mkdir(RELEASE_PROGRESS)

start_release()