From 2b9b5f3ca72ecfa4a5c1801a7fd32e3bb854bef2 Mon Sep 17 00:00:00 2001 From: Nathan Bolam Date: Sun, 24 Jul 2022 08:56:14 -0500 Subject: [PATCH] #3 git remote helper for IPFS --- git-remote-helper/.devcontainer/Dockerfile | 18 ++ .../.devcontainer/devcontainer.json | 58 +++++ git-remote-helper/.gitignore | 24 ++ git-remote-helper/.vscode/launch.json | 12 + git-remote-helper/README.md | 26 ++ git-remote-helper/fetch.go | 158 ++++++++++++ git-remote-helper/git.go | 135 ++++++++++ git-remote-helper/go.mod | 14 ++ git-remote-helper/go.sum | 128 ++++++++++ git-remote-helper/helpers.go | 31 +++ git-remote-helper/internal/path/path.go | 111 +++++++++ git-remote-helper/list.go | 131 ++++++++++ git-remote-helper/main.go | 234 ++++++++++++++++++ git-remote-helper/push.go | 117 +++++++++ 14 files changed, 1197 insertions(+) create mode 100644 git-remote-helper/.devcontainer/Dockerfile create mode 100644 git-remote-helper/.devcontainer/devcontainer.json create mode 100644 git-remote-helper/.gitignore create mode 100644 git-remote-helper/.vscode/launch.json create mode 100644 git-remote-helper/README.md create mode 100644 git-remote-helper/fetch.go create mode 100644 git-remote-helper/git.go create mode 100644 git-remote-helper/go.mod create mode 100644 git-remote-helper/go.sum create mode 100644 git-remote-helper/helpers.go create mode 100644 git-remote-helper/internal/path/path.go create mode 100644 git-remote-helper/list.go create mode 100644 git-remote-helper/main.go create mode 100644 git-remote-helper/push.go diff --git a/git-remote-helper/.devcontainer/Dockerfile b/git-remote-helper/.devcontainer/Dockerfile new file mode 100644 index 0000000..433176a --- /dev/null +++ b/git-remote-helper/.devcontainer/Dockerfile @@ -0,0 +1,18 @@ +# See here for image contents: https://github.com/microsoft/vscode-dev-containers/tree/v0.195.0/containers/go/.devcontainer/base.Dockerfile +# [Choice] Go version (use -bullseye variants on local arm64/Apple Silicon): 1, 1.16, 1.17, 1-bullseye, 1.16-bullseye, 1.17-bullseye, 1-buster, 1.16-buster, 1.17-buster +ARG VARIANT=1-bullseye +FROM mcr.microsoft.com/vscode/devcontainers/go:0-${VARIANT} + +# [Choice] Node.js version: lts/*, 16, 14, 12, 10 +ARG NODE_VERSION="lts/*" +RUN if [ "${NODE_VERSION}" != "none" ]; then su vscode -c ". /usr/local/share/nvm/nvm.sh && nvm install ${NODE_VERSION} 2>&1"; fi + +# [Optional] Uncomment this section to install additional OS packages. +# RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \ +# && apt-get -y install --no-install-recommends + +# [Optional] Uncomment the next line to use go get to install anything else you need +# RUN go get -x + +# [Optional] Uncomment this line to install global node packages. +# RUN su vscode -c "source /usr/local/share/nvm/nvm.sh && npm install -g " 2>&1 diff --git a/git-remote-helper/.devcontainer/devcontainer.json b/git-remote-helper/.devcontainer/devcontainer.json new file mode 100644 index 0000000..3a7235f --- /dev/null +++ b/git-remote-helper/.devcontainer/devcontainer.json @@ -0,0 +1,58 @@ +// For format details, see https://aka.ms/vscode-remote/devcontainer.json or this file's README at: +// https://github.com/microsoft/vscode-dev-containers/tree/v0.195.0/containers/go +{ + "name": "Go", + "build": { + "dockerfile": "Dockerfile", + "args": { + // Update the VARIANT arg to pick a version of Go: 1, 1.16, 1.17 + // Append -bullseye or -buster to pin to an OS version. + // Use -bullseye variants on local arm64/Apple Silicon. + "VARIANT": "1.17-bullseye", + // Options + "NODE_VERSION": "lts/*" + } + }, + "runArgs": [ "--cap-add=SYS_PTRACE", "--security-opt", "seccomp=unconfined" ], + + // Configure tool-specific properties. + "customizations": { + // Configure properties specific to VS Code. + "vscode": { + // Set *default* container specific settings.json values on container create. + "settings": { + "go.toolsManagement.checkForUpdates": "local", + "go.useLanguageServer": true, + "go.gopath": "/go", + "go.goroot": "/usr/local/go" + }, + + // Add the IDs of extensions you want installed when the container is created. + "extensions": [ + "golang.Go" + ] + } + }, + + // Use 'forwardPorts' to make a list of ports inside the container available locally. + // "forwardPorts": [9000], + + // Use 'portsAttributes' to set default properties for specific forwarded ports. More info: https://code.visualstudio.com/docs/remote/devcontainerjson-reference. + "portsAttributes": { + "9000": { + "label": "Hello Remote World", + "onAutoForward": "notify" + } + }, + + // Use 'otherPortsAttributes' to configure any ports that aren't configured using 'portsAttributes'. + // "otherPortsAttributes": { + // "onAutoForward": "silent" + // }, + + // Use 'postCreateCommand' to run commands after the container is created. + // "postCreateCommand": "go version", + + // Uncomment to connect as a non-root user. More info: https://aka.ms/vscode-remote/containers/non-root. + "remoteUser": "vscode" +} diff --git a/git-remote-helper/.gitignore b/git-remote-helper/.gitignore new file mode 100644 index 0000000..daf913b --- /dev/null +++ b/git-remote-helper/.gitignore @@ -0,0 +1,24 @@ +# Compiled Object files, Static and Dynamic libs (Shared Objects) +*.o +*.a +*.so + +# Folders +_obj +_test + +# Architecture specific extensions/prefixes +*.[568vq] +[568vq].out + +*.cgo1.go +*.cgo2.c +_cgo_defun.c +_cgo_gotypes.go +_cgo_export.* + +_testmain.go + +*.exe +*.test +*.prof diff --git a/git-remote-helper/.vscode/launch.json b/git-remote-helper/.vscode/launch.json new file mode 100644 index 0000000..1a991ed --- /dev/null +++ b/git-remote-helper/.vscode/launch.json @@ -0,0 +1,12 @@ +{ + "version": "0.2.0", + "configurations": [ + { + "name": "Launch Server", + "type": "go", + "request": "launch", + "mode": "debug", + "program": "${workspaceFolder}/server.go" + } + ] +} diff --git a/git-remote-helper/README.md b/git-remote-helper/README.md new file mode 100644 index 0000000..63621e5 --- /dev/null +++ b/git-remote-helper/README.md @@ -0,0 +1,26 @@ +# Gitblox Remote Helper + +`git-remote-helper` implements a git-remote helper that uses the ipfs transport. + +### TODO + +Currently assumes a IPFS Daemon at localhost:5001 + + +### Usage + +``` +git clone gitblox://ipfs/$hash/repo.git +cd repo && make $stuff +git commit -a -m 'done!' +git push origin +``` + +### Links + +- https://ipfs.io +- https://github.com/whyrusleeping/git-ipfs-rehost +- https://git-scm.com/docs/gitremote-helpers +- https://git-scm.com/book/en/v2/Git-Internals-Plumbing-and-Porcelain +- https://git-scm.com/docs/gitrepository-layout +- https://git-scm.com/book/en/v2/Git-Internals-Transfer-Protocols \ No newline at end of file diff --git a/git-remote-helper/fetch.go b/git-remote-helper/fetch.go new file mode 100644 index 0000000..8685cca --- /dev/null +++ b/git-remote-helper/fetch.go @@ -0,0 +1,158 @@ +package main + +import ( + "bytes" + "io" + "os" + "os/exec" + "path/filepath" + "strings" + + "github.com/cryptix/exp/git" + "github.com/pkg/errors" +) + +// "fetch $sha1 $ref" method 1 - unpacking loose objects +// - look for it in ".git/objects/substr($sha1, 0, 2)/substr($sha, 2)" +// - if found, download it and put it in place. (there may be a command for this) +// - done \o/ +func fetchObject(sha1 string) error { + return recurseCommit(sha1) +} + +func recurseCommit(sha1 string) error { + obj, err := fetchAndWriteObj(sha1) + if err != nil { + return errors.Wrapf(err, "fetchAndWriteObj(%s) commit object failed", sha1) + } + commit, ok := obj.Commit() + if !ok { + return errors.Errorf("sha1<%s> is not a git commit object:%s ", sha1, obj) + } + if commit.Parent != "" { + if err := recurseCommit(commit.Parent); err != nil { + return errors.Wrapf(err, "recurseCommit(%s) commit Parent failed", commit.Parent) + } + } + return fetchTree(commit.Tree) +} + +func fetchTree(sha1 string) error { + obj, err := fetchAndWriteObj(sha1) + if err != nil { + return errors.Wrapf(err, "fetchAndWriteObj(%s) commit tree failed", sha1) + } + entries, ok := obj.Tree() + if !ok { + return errors.Errorf("sha1<%s> is not a git tree object:%s ", sha1, obj) + } + for _, t := range entries { + obj, err := fetchAndWriteObj(t.SHA1Sum.String()) + if err != nil { + return errors.Wrapf(err, "fetchAndWriteObj(%s) commit tree failed", sha1) + } + if obj.Type != git.BlobT { + return errors.Errorf("sha1<%s> is not a git tree object:%s ", t.SHA1Sum.String(), obj) + } + } + return nil +} + +// fetchAndWriteObj looks for the loose object under 'thisGitRepo' global git dir +// and usses an io.TeeReader to write it to the local repo +func fetchAndWriteObj(sha1 string) (*git.Object, error) { + p := filepath.Join(ipfsRepoPath, "objects", sha1[:2], sha1[2:]) + ipfsCat, err := ipfsShell.Cat(p) + if err != nil { + return nil, errors.Wrapf(err, "shell.Cat() commit failed") + } + targetP := filepath.Join(thisGitRepo, "objects", sha1[:2], sha1[2:]) + if err := os.MkdirAll(filepath.Join(thisGitRepo, "objects", sha1[:2]), 0700); err != nil { + return nil, errors.Wrapf(err, "mkDirAll() failed") + } + targetObj, err := os.Create(targetP) + if err != nil { + return nil, errors.Wrapf(err, "os.Create(%s) commit failed", targetP) + } + obj, err := git.DecodeObject(io.TeeReader(ipfsCat, targetObj)) + if err != nil { + return nil, errors.Wrapf(err, "git.DecodeObject(commit) failed") + } + + if err := ipfsCat.Close(); err != nil { + err = errors.Wrap(err, "ipfs/cat Close failed") + if errRm := os.Remove(targetObj.Name()); errRm != nil { + err = errors.Wrapf(err, "failed removing targetObj: %s", errRm) + return nil, err + } + return nil, errors.Wrapf(err, "closing ipfs cat failed") + } + + if err := targetObj.Close(); err != nil { + return nil, errors.Wrapf(err, "target file close() failed") + } + + return obj, nil +} + +// "fetch $sha1 $ref" method 2 - unpacking packed objects +// - look for it in packfiles by fetching ".git/objects/pack/*.idx" +// and looking at each idx with cat | git show-index (alternatively can learn to read the format in go) +// - if found in an , download the relevant .pack file, +// and feed it into `git index-pack --stdin --fix-thin` which will put it into place. +// - done \o/ +func fetchPackedObject(sha1 string) error { + // search for all index files + packPath := filepath.Join(ipfsRepoPath, "objects", "pack") + links, err := ipfsShell.List(packPath) + if err != nil { + return errors.Wrapf(err, "shell FileList(%q) failed", packPath) + } + var indexes []string + for _, lnk := range links { + if lnk.Type == 2 && strings.HasSuffix(lnk.Name, ".idx") { + indexes = append(indexes, filepath.Join(packPath, lnk.Name)) + } + } + if len(indexes) == 0 { + return errors.New("fetchPackedObject: no idx files found") + } + for _, idx := range indexes { + idxF, err := ipfsShell.Cat(idx) + if err != nil { + return errors.Wrapf(err, "fetchPackedObject: idx<%s> cat(%s) failed", sha1, idx) + } + // using external git show-index < idxF for now + // TODO: parse index file in go to make this portable + var b bytes.Buffer + showIdx := exec.Command("git", "show-index") + showIdx.Stdin = idxF + showIdx.Stdout = &b + showIdx.Stderr = &b + if err := showIdx.Run(); err != nil { + return errors.Wrapf(err, "fetchPackedObject: idx<%s> show-index start failed", sha1) + } + cmdOut := b.String() + if !strings.Contains(cmdOut, sha1) { + log.Log("idx", filepath.Base(idx), "event", "debug", "msg", "git show-index: sha1 not in index, next idx file") + continue + } + // we found an index with our hash inside + pack := strings.Replace(idx, ".idx", ".pack", 1) + packF, err := ipfsShell.Cat(pack) + if err != nil { + return errors.Wrapf(err, "fetchPackedObject: pack<%s> open() failed", sha1) + } + b.Reset() + unpackIdx := exec.Command("git", "unpack-objects") + unpackIdx.Dir = thisGitRepo // GIT_DIR + unpackIdx.Stdin = packF + unpackIdx.Stdout = &b + unpackIdx.Stderr = &b + if err := unpackIdx.Run(); err != nil { + return errors.Wrapf(err, "fetchPackedObject: pack<%s> 'git unpack-objects' failed\nOutput: %s", sha1, b.String()) + } + return nil + } + return errors.Errorf("did not find sha1<%s> in %d index files", sha1, len(indexes)) +} diff --git a/git-remote-helper/git.go b/git-remote-helper/git.go new file mode 100644 index 0000000..84c03b7 --- /dev/null +++ b/git-remote-helper/git.go @@ -0,0 +1,135 @@ +package main + +import ( + "bufio" + "bytes" + "compress/zlib" + "fmt" + "io" + "io/ioutil" + "os/exec" + "path/filepath" + "strconv" + "strings" + + "github.com/pkg/errors" +) + +// return the objects reachable from ref excluding the objects reachable from exclude +func gitListObjects(ref string, exclude []string) ([]string, error) { + args := []string{"rev-list", "--objects", ref} + for _, e := range exclude { + args = append(args, "^"+e) + } + revList := exec.Command("git", args...) + // dunno why - sometime git doesnt want to work on the inner repo/.git + if strings.HasSuffix(thisGitRepo, ".git") { + thisGitRepo = filepath.Dir(thisGitRepo) + } + revList.Dir = thisGitRepo // GIT_DIR + out, err := revList.CombinedOutput() + if err != nil { + return nil, errors.Wrapf(err, "rev-list failed: %s\n%q", err, string(out)) + } + var objs []string + s := bufio.NewScanner(bytes.NewReader(out)) + for s.Scan() { + objs = append(objs, strings.Split(s.Text(), " ")[0]) + } + if err := s.Err(); err != nil { + return nil, errors.Wrapf(err, "scanning rev-list output failed: %s", err) + } + return objs, nil +} + +func gitFlattenObject(sha1 string) (io.Reader, error) { + kind, err := gitCatKind(sha1) + if err != nil { + return nil, errors.Wrapf(err, "flatten: kind(%s) failed", sha1) + } + size, err := gitCatSize(sha1) + if err != nil { + return nil, errors.Wrapf(err, "flatten: size(%s) failed", sha1) + } + r, err := gitCatData(sha1, kind) + if err != nil { + return nil, errors.Wrapf(err, "flatten: data(%s) failed", sha1) + } + // move to exp/git + pr, pw := io.Pipe() + go func() { + zw := zlib.NewWriter(pw) + if _, err := fmt.Fprintf(zw, "%s %d\x00", kind, size); err != nil { + pw.CloseWithError(errors.Wrapf(err, "writing git format header failed")) + return + } + if _, err := io.Copy(zw, r); err != nil { + pw.CloseWithError(errors.Wrapf(err, "copying git data failed")) + return + } + if err := zw.Close(); err != nil { + pw.CloseWithError(errors.Wrapf(err, "zlib close failed")) + return + } + pw.Close() + }() + return pr, nil +} + +func gitCatKind(sha1 string) (string, error) { + catFile := exec.Command("git", "cat-file", "-t", sha1) + catFile.Dir = thisGitRepo // GIT_DIR + out, err := catFile.CombinedOutput() + return strings.TrimSpace(string(out)), err +} + +func gitCatSize(sha1 string) (int64, error) { + catFile := exec.Command("git", "cat-file", "-s", sha1) + catFile.Dir = thisGitRepo // GIT_DIR + out, err := catFile.CombinedOutput() + if err != nil { + return -1, errors.Wrapf(err, "catSize(%s): run failed", sha1) + } + return strconv.ParseInt(strings.TrimSpace(string(out)), 10, 64) +} + +func gitCatData(sha1, kind string) (io.Reader, error) { + catFile := exec.Command("git", "cat-file", kind, sha1) + catFile.Dir = thisGitRepo // GIT_DIR + stdout, err := catFile.StdoutPipe() + if err != nil { + return nil, errors.Wrapf(err, "catData(%s): stdoutPipe failed", sha1) + } + stderr, err := catFile.StderrPipe() + if err != nil { + return nil, errors.Wrapf(err, "catData(%s): stderrPipe failed", sha1) + } + r := io.MultiReader(stdout, stderr) + if err := catFile.Start(); err != nil { + err = errors.Wrap(err, "catFile.Start failed") + out, readErr := ioutil.ReadAll(r) + if readErr != nil { + readErr = errors.Wrap(readErr, "readAll failed") + return nil, errors.Wrapf(err, "catData(%s) failed during: %s", sha1, readErr) + } + return nil, errors.Wrapf(err, "catData(%s) failed: %q", sha1, out) + } + // todo wait for cmd?! + return r, nil +} + +func gitRefHash(ref string) (string, error) { + refParse := exec.Command("git", "rev-parse", ref) + refParse.Dir = thisGitRepo // GIT_DIR + out, err := refParse.CombinedOutput() + return strings.TrimSpace(string(out)), err +} + +func gitIsAncestor(a, ref string) error { + mergeBase := exec.Command("git", "merge-base", "--is-ancestor", a, ref) + mergeBase.Dir = thisGitRepo // GIT_DIR + if out, err := mergeBase.CombinedOutput(); err != nil { + return errors.Wrapf(err, "merge-base failed: %q", string(out)) + } + return nil +} diff --git a/git-remote-helper/go.mod b/git-remote-helper/go.mod new file mode 100644 index 0000000..f36a748 --- /dev/null +++ b/git-remote-helper/go.mod @@ -0,0 +1,14 @@ +module github.com/capturealpha/gitblox/git-remote-gitblox + +go 1.13 + +require ( + github.com/cryptix/exp v0.0.0-20191103140156-9da154296953 + github.com/cryptix/go v1.5.0 + github.com/ipfs/go-cid v0.0.3 + github.com/ipfs/go-ipfs-api v0.0.2 + github.com/jbenet/go-random v0.0.0-20190219211222-123a90aedc0c + github.com/kylelemons/godebug v1.1.0 // indirect + github.com/multiformats/go-multihash v0.0.9 + github.com/pkg/errors v0.8.1 +) diff --git a/git-remote-helper/go.sum b/git-remote-helper/go.sum new file mode 100644 index 0000000..0e0dfef --- /dev/null +++ b/git-remote-helper/go.sum @@ -0,0 +1,128 @@ +github.com/PuerkitoBio/goquery v1.5.0/go.mod h1:qD2PgZ9lccMbQlc7eEOjaeRlFQON7xY8kdmcsrnKqMg= +github.com/aead/siphash v1.0.1/go.mod h1:Nywa3cDsYNNK3gaciGTWPwHt0wlpNV15vwmswBAUSII= +github.com/andybalholm/cascadia v1.0.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y= +github.com/btcsuite/btcd v0.0.0-20190213025234-306aecffea32 h1:qkOC5Gd33k54tobS36cXdAzJbeHaduLtnLQQwNoIi78= +github.com/btcsuite/btcd v0.0.0-20190213025234-306aecffea32/go.mod h1:DrZx5ec/dmnfpw9KyYoQyYo7d0KEvTkk/5M/vbZjAr8= +github.com/btcsuite/btclog v0.0.0-20170628155309-84c8d2346e9f/go.mod h1:TdznJufoqS23FtqVCzL0ZqgP5MqXbb4fg/WgDys70nA= +github.com/btcsuite/btcutil v0.0.0-20190207003914-4c204d697803/go.mod h1:+5NJ2+qvTyV9exUAL/rxXi3DcLg2Ts+ymUAY5y4NvMg= +github.com/btcsuite/go-socks v0.0.0-20170105172521-4720035b7bfd/go.mod h1:HHNXQzUsZCxOoE+CPiyCTO6x34Zs86zZUiwtpXoGdtg= +github.com/btcsuite/goleveldb v0.0.0-20160330041536-7834afc9e8cd/go.mod h1:F+uVaaLLH7j4eDXPRvw78tMflu7Ie2bzYOH4Y8rRKBY= +github.com/btcsuite/snappy-go v0.0.0-20151229074030-0bdef8d06723/go.mod h1:8woku9dyThutzjeg+3xrA5iCpBRH8XEEg3lh6TiUghc= +github.com/btcsuite/websocket v0.0.0-20150119174127-31079b680792/go.mod h1:ghJtEyQwv5/p4Mg4C0fgbePVuGr935/5ddU9Z3TmDRY= +github.com/btcsuite/winsvc v1.0.0/go.mod h1:jsenWakMcC0zFBFurPLEAyrnc/teJEM1O46fmI40EZs= +github.com/cheekybits/is v0.0.0-20150225183255-68e9c0620927 h1:SKI1/fuSdodxmNNyVBR8d7X/HuLnRpvvFO0AgyQk764= +github.com/cheekybits/is v0.0.0-20150225183255-68e9c0620927/go.mod h1:h/aW8ynjgkuj+NQRlZcDbAbM1ORAbXjXX77sX7T289U= +github.com/cryptix/exp v0.0.0-20191103140156-9da154296953 h1:1y5SKCkkuOZiadIdH37xDTsY9SGvWV4vy0jSWgT9m+c= +github.com/cryptix/exp v0.0.0-20191103140156-9da154296953/go.mod h1:+3mt2noPsYFTh4fa/S6YnM/NsS5nZAoCofDSZi77YrQ= +github.com/cryptix/go v1.5.0 h1:2+g9oKxiTc5ELGISY2E4Q4E7X8FwP3cXyKbb93EUaZY= +github.com/cryptix/go v1.5.0/go.mod h1:bopBFzjGB7Oqrsvt7HaPZSOX1gJW7hgIb84XNhnOhOY= +github.com/davecgh/go-spew v0.0.0-20171005155431-ecdeabc65495/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= +github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/go-kit/kit v0.8.0 h1:Wz+5lgoB0kkuqLEc6NVmwRknTKP6dTGbSqvhZtBI/j0= +github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-logfmt/logfmt v0.4.0 h1:MP4Eh7ZCb31lleYCFuwm0oe4/YGak+5l1vA2NOE80nA= +github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= +github.com/go-stack/stack v1.8.0 h1:5SgMzNM5HxrEjV0ww2lTmX6E2Izsfxas4+YHWRs3Lsk= +github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/gogo/protobuf v1.2.1 h1:/s5zKNz0uPFCZ5hddgPdo2TK2TVrUNMn0OOX8/aZMTE= +github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg= +github.com/gorilla/securecookie v1.1.1/go.mod h1:ra0sb63/xPlUeL+yeDciTfxMRAA+MP+HVt/4epWDjd4= +github.com/gorilla/sessions v1.1.3/go.mod h1:8KCfur6+4Mqcc6S0FEfKuN15Vl5MgXW92AE8ovaJD0w= +github.com/gxed/hashland/keccakpg v0.0.1/go.mod h1:kRzw3HkwxFU1mpmPP8v1WyQzwdGfmKFJ6tItnhQ67kU= +github.com/gxed/hashland/murmur3 v0.0.1/go.mod h1:KjXop02n4/ckmZSnY2+HKcLud/tcmvhST0bie/0lS48= +github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= +github.com/ipfs/go-cid v0.0.3 h1:UIAh32wymBpStoe83YCzwVQQ5Oy/H0FdxvUS6DJDzms= +github.com/ipfs/go-cid v0.0.3/go.mod h1:GHWU/WuQdMPmIosc4Yn1bcCT7dSeX4lBafM7iqUPQvM= +github.com/ipfs/go-ipfs-api v0.0.2 h1:Yg+c8oAMVrN1m0OUGfrWBF+rhfqb7uJ123gU2g1giec= +github.com/ipfs/go-ipfs-api v0.0.2/go.mod h1:0FhXgCzrLu7qNmdxZvgYqD9jFzJxzz1NAVt3OQ0WOIc= +github.com/ipfs/go-ipfs-files v0.0.1 h1:OroTsI58plHGX70HPLKy6LQhPR3HZJ5ip61fYlo6POM= +github.com/ipfs/go-ipfs-files v0.0.1/go.mod h1:INEFm0LL2LWXBhNJ2PMIIb2w45hpXgPjNoE7yA8Y1d4= +github.com/ipfs/go-ipfs-util v0.0.1/go.mod h1:spsl5z8KUnrve+73pOhSVZND1SIxPW5RyBCNzQxlJBc= +github.com/jbenet/go-random v0.0.0-20190219211222-123a90aedc0c h1:uUx61FiAa1GI6ZmVd2wf2vULeQZIKG66eybjNXKYCz4= +github.com/jbenet/go-random v0.0.0-20190219211222-123a90aedc0c/go.mod h1:sdx1xVM9UuLw1tXnhJWN3piypTUO3vCIHYmG15KE/dU= +github.com/jessevdk/go-flags v0.0.0-20141203071132-1679536dcc89/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= +github.com/jrick/logrotate v1.0.0/go.mod h1:LNinyqDIJnpAur+b8yyulnQw/wDuN1+BYKlTRt3OuAQ= +github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23/go.mod h1:J+Gs4SYgM6CZQHDETBtE9HaSEkGmuNXF86RwHhHUvq4= +github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515 h1:T+h1c/A9Gawja4Y9mFVWj2vyii2bbUNDw3kt9VxK2EY= +github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= +github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= +github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= +github.com/libp2p/go-flow-metrics v0.0.1 h1:0gxuFd2GuK7IIP5pKljLwps6TvcuYgvG7Atqi3INF5s= +github.com/libp2p/go-flow-metrics v0.0.1/go.mod h1:Iv1GH0sG8DtYN3SVJ2eG221wMiNpZxBdp967ls1g+k8= +github.com/libp2p/go-libp2p-crypto v0.0.1 h1:JNQd8CmoGTohO/akqrH16ewsqZpci2CbgYH/LmYl8gw= +github.com/libp2p/go-libp2p-crypto v0.0.1/go.mod h1:yJkNyDmO341d5wwXxDUGO0LykUVT72ImHNUqh5D/dBE= +github.com/libp2p/go-libp2p-metrics v0.0.1 h1:yumdPC/P2VzINdmcKZd0pciSUCpou+s0lwYCjBbzQZU= +github.com/libp2p/go-libp2p-metrics v0.0.1/go.mod h1:jQJ95SXXA/K1VZi13h52WZMa9ja78zjyy5rspMsC/08= +github.com/libp2p/go-libp2p-peer v0.0.1 h1:0qwAOljzYewINrU+Kndoc+1jAL7vzY/oY2Go4DCGfyY= +github.com/libp2p/go-libp2p-peer v0.0.1/go.mod h1:nXQvOBbwVqoP+T5Y5nCjeH4sP9IX/J0AMzcDUVruVoo= +github.com/libp2p/go-libp2p-protocol v0.0.1 h1:+zkEmZ2yFDi5adpVE3t9dqh/N9TbpFWywowzeEzBbLM= +github.com/libp2p/go-libp2p-protocol v0.0.1/go.mod h1:Af9n4PiruirSDjHycM1QuiMi/1VZNHYcK8cLgFJLZ4s= +github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1 h1:lYpkrQH5ajf0OXOcUbGjvZxxijuBwbbmlSxLiuofa+g= +github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1/go.mod h1:pD8RvIylQ358TN4wwqatJ8rNavkEINozVn9DtGI3dfQ= +github.com/minio/sha256-simd v0.0.0-20190131020904-2d45a736cd16/go.mod h1:2FMWW+8GMoPweT6+pI63m9YE3Lmw4J71hV56Chs1E/U= +github.com/minio/sha256-simd v0.1.1-0.20190913151208-6de447530771 h1:MHkK1uRtFbVqvAgvWxafZe54+5uBxLluGylDiKgdhwo= +github.com/minio/sha256-simd v0.1.1-0.20190913151208-6de447530771/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM= +github.com/miolini/datacounter v0.0.0-20171104152933-fd4e42a1d5e0/go.mod h1:P6fDJzlxN+cWYR09KbE9/ta+Y6JofX9tAUhJpWkWPaM= +github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= +github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mr-tron/base58 v1.1.0/go.mod h1:xcD2VGqlgYjBdcBLw+TuYLr8afG+Hj8g2eTVqeSzSU8= +github.com/mr-tron/base58 v1.1.2 h1:ZEw4I2EgPKDJ2iEw0cNmLB3ROrEmkOtXIkaG7wZg+78= +github.com/mr-tron/base58 v1.1.2/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc= +github.com/multiformats/go-base32 v0.0.3 h1:tw5+NhuwaOjJCC5Pp82QuXbrmLzWg7uxlMFp8Nq/kkI= +github.com/multiformats/go-base32 v0.0.3/go.mod h1:pLiuGC8y0QR3Ue4Zug5UzK9LjgbkL8NSQj0zQ5Nz/AA= +github.com/multiformats/go-multiaddr v0.0.1 h1:/QUV3VBMDI6pi6xfiw7lr6xhDWWvQKn9udPn68kLSdY= +github.com/multiformats/go-multiaddr v0.0.1/go.mod h1:xKVEak1K9cS1VdmPZW3LSIb6lgmoS58qz/pzqmAxV44= +github.com/multiformats/go-multiaddr-dns v0.0.1 h1:jQt9c6tDSdQLIlBo4tXYx7QUHCPjxsB1zXcag/2S7zc= +github.com/multiformats/go-multiaddr-dns v0.0.1/go.mod h1:9kWcqw/Pj6FwxAwW38n/9403szc57zJPs45fmnznu3Q= +github.com/multiformats/go-multiaddr-net v0.0.1 h1:76O59E3FavvHqNg7jvzWzsPSW5JSi/ek0E4eiDVbg9g= +github.com/multiformats/go-multiaddr-net v0.0.1/go.mod h1:nw6HSxNmCIQH27XPGBuX+d1tnvM7ihcFwHMSstNAVUU= +github.com/multiformats/go-multibase v0.0.1 h1:PN9/v21eLywrFWdFNsFKaU04kLJzuYzmrJR+ubhT9qA= +github.com/multiformats/go-multibase v0.0.1/go.mod h1:bja2MqRZ3ggyXtZSEDKpl0uO/gviWFaSteVbWT51qgs= +github.com/multiformats/go-multihash v0.0.1/go.mod h1:w/5tugSrLEbWqlcgJabL3oHFKTwfvkofsjW2Qa1ct4U= +github.com/multiformats/go-multihash v0.0.9 h1:aoijQXYYl7Xtb2pUUP68R+ys1TlnlR3eX6wmozr0Hp4= +github.com/multiformats/go-multihash v0.0.9/go.mod h1:YSLudS+Pi8NHE7o6tb3D8vrpKa63epEDmG8nTduyAew= +github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= +github.com/oxtoacart/bpool v0.0.0-20190524125616-8c0b41497736/go.mod h1:L3UMQOThbttwfYRNFOWLLVXMhk5Lkio4GGOtw5UrxS0= +github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= +github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/shurcooL/httpfs v0.0.0-20190527155220-6a4d4a70508b/go.mod h1:ZY1cvUeJuFPAdZ/B6v7RHavJWZn2YPVFQ1OSXhCGOkg= +github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= +github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/whyrusleeping/tar-utils v0.0.0-20180509141711-8c6c8ba81d5c h1:GGsyl0dZ2jJgVT+VvWBf/cNijrHRhkrTjkmp5wg7li0= +github.com/whyrusleeping/tar-utils v0.0.0-20180509141711-8c6c8ba81d5c/go.mod h1:xxcJeBb7SIUl/Wzkz1eVKJE/CB34YNrqX2TQI6jY9zs= +golang.org/x/crypto v0.0.0-20170930174604-9419663f5a44/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20190211182817-74369b46fc67/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20190225124518-7f87c0fbb88b/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8 h1:1wopBVtVdWnn03fZelqdXTqk7U7zPQCb+T4rbU9ZEoU= +golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190219092855-153ac476189d/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190302025703-b6889370fb10/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d h1:+R4KGOnez64A81RvjARKc4UT5/tI9ujCIVX+P5KiHuI= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= +gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/git-remote-helper/helpers.go b/git-remote-helper/helpers.go new file mode 100644 index 0000000..776ebad --- /dev/null +++ b/git-remote-helper/helpers.go @@ -0,0 +1,31 @@ +package main + +import ( + "os" + "os/signal" + "path/filepath" + "syscall" + + "github.com/pkg/errors" +) + +func fetchFullBareRepo(root string) (string, error) { + // TODO: get host from envvar + tmpPath := filepath.Join("/", os.TempDir(), root) + _, err := os.Stat(tmpPath) + switch { + case os.IsNotExist(err) || err == nil: + if err := ipfsShell.Get(root, tmpPath); err != nil { + return "", errors.Wrapf(err, "shell.Get(%s, %s) failed: %s", root, tmpPath, err) + } + return tmpPath, nil + default: + return "", errors.Wrap(err, "os.Stat(): unhandled error") + } +} + +func interrupt() error { + c := make(chan os.Signal) + signal.Notify(c, syscall.SIGINT, syscall.SIGTERM) + return errors.Errorf("%s", <-c) +} diff --git a/git-remote-helper/internal/path/path.go b/git-remote-helper/internal/path/path.go new file mode 100644 index 0000000..aa371e0 --- /dev/null +++ b/git-remote-helper/internal/path/path.go @@ -0,0 +1,111 @@ +package path + +import ( + "errors" + "fmt" + "path" + "strings" + + "github.com/ipfs/go-cid" + mh "github.com/multiformats/go-multihash" +) + +// ErrBadPath is returned when a given path is incorrectly formatted +var ErrBadPath = errors.New("invalid ipfs ref path") + +// TODO: debate making this a private struct wrapped in a public interface +// would allow us to control creation, and cache segments. +type Path string + +// FromString safely converts a string type to a Path type +func FromString(s string) Path { + return Path(s) +} + +func (p Path) Segments() []string { + cleaned := path.Clean(string(p)) + segments := strings.Split(cleaned, "/") + + // Ignore leading slash + if len(segments[0]) == 0 { + segments = segments[1:] + } + + return segments +} + +func (p Path) String() string { + return string(p) +} + +// FromCid safely converts a cid.Cid type to a Path type +func FromCid(c *cid.Cid) Path { + return Path("/ipfs/" + c.String()) +} +func FromSegments(prefix string, seg ...string) (Path, error) { + return ParsePath(prefix + strings.Join(seg, "/")) +} + +func ParsePath(txt string) (Path, error) { + fmt.Println(txt) + parts := strings.Split(txt, "/") + fmt.Println(len(parts)) + if len(parts) == 1 { + kp, err := ParseCidToPath(txt) + if err == nil { + return kp, nil + } + } + + // if the path doesnt being with a '/' + // we expect this to start with a hash, and be an 'ipfs' path + if parts[0] != "" { + if _, err := ParseCidToPath(parts[0]); err != nil { + return "", ErrBadPath + } + // The case when the path starts with hash without a protocol prefix + return Path("/ipfs/" + txt), nil + } + + if len(parts) < 3 { + return "", ErrBadPath + } + + if parts[1] == "ipfs" { + if _, err := ParseCidToPath(parts[2]); err != nil { + return "", err + } + } else if parts[1] != "ipns" { + return "", ErrBadPath + } + + return Path(txt), nil +} + +func ParseCidToPath(txt string) (Path, error) { + if txt == "" { + return "", ErrNoComponents + } + + c, err := cid.Decode(txt) + if err != nil { + return "", err + } + + return FromCid(&c), nil +} + +func (p *Path) IsValid() error { + _, err := ParsePath(p.String()) + return err +} + +// Paths after a protocol must contain at least one component +var ErrNoComponents = errors.New( + "path must contain at least one component") + +// ErrNoLink is returned when a link is not found in a path +type ErrNoLink struct { + Name string + Node mh.Multihash +} diff --git a/git-remote-helper/list.go b/git-remote-helper/list.go new file mode 100644 index 0000000..6203fac --- /dev/null +++ b/git-remote-helper/list.go @@ -0,0 +1,131 @@ +package main + +import ( + "bufio" + "bytes" + "io/ioutil" + "path/filepath" + "strings" + + shell "github.com/ipfs/go-ipfs-api" + "github.com/pkg/errors" +) + +func listInfoRefs(forPush bool) error { + log.Log(filepath.Join(ipfsRepoPath, "info", "refs")) + refsCat, err := ipfsShell.Cat(filepath.Join(ipfsRepoPath, "info", "refs")) + if err != nil { + return errors.Wrapf(err, "failed to cat info/refs from %s", ipfsRepoPath) + } + s := bufio.NewScanner(refsCat) + for s.Scan() { + hashRef := strings.Split(s.Text(), "\t") + if len(hashRef) != 2 { + return errors.Errorf("processing info/refs: what is this: %v", hashRef) + } + ref2hash[hashRef[1]] = hashRef[0] + } + if err := s.Err(); err != nil { + return errors.Wrapf(err, "ipfs.Cat(info/refs) scanner error") + } + return nil +} + +func listHeadRef() (string, error) { + headCat, err := ipfsShell.Cat(filepath.Join(ipfsRepoPath, "HEAD")) + if err != nil { + return "", errors.Wrapf(err, "failed to cat HEAD from %s", ipfsRepoPath) + } + head, err := ioutil.ReadAll(headCat) + if err != nil { + return "", errors.Wrapf(err, "failed to readAll HEAD from %s", ipfsRepoPath) + } + if !bytes.HasPrefix(head, []byte("ref: ")) { + return "", errors.Errorf("illegal HEAD file from %s: %q", ipfsRepoPath, head) + } + headRef := string(bytes.TrimSpace(head[5:])) + headHash, ok := ref2hash[headRef] + if !ok { + // use first hash in map?.. + return "", errors.Errorf("unknown HEAD reference %q", headRef) + } + return headHash, headCat.Close() +} + +func listIterateRefs(forPush bool) error { + refsDir := filepath.Join(ipfsRepoPath, "refs") + return Walk(refsDir, func(p string, info *shell.LsLink, err error) error { + if err != nil { + return errors.Wrapf(err, "walk(%s) failed", p) + } + log.Log("event", "debug", "name", info.Name, "msg", "iterateRefs: walked to", "p", p) + if info.Type == 2 { + rc, err := ipfsShell.Cat(p) + if err != nil { + return errors.Wrapf(err, "walk(%s) cat ref failed", p) + } + data, err := ioutil.ReadAll(rc) + if err != nil { + return errors.Wrapf(err, "walk(%s) readAll failed", p) + } + if err := rc.Close(); err != nil { + return errors.Wrapf(err, "walk(%s) cat close failed", p) + } + sha1 := strings.TrimSpace(string(data)) + refName := strings.TrimPrefix(p, ipfsRepoPath+"/") + ref2hash[refName] = sha1 + log.Log("event", "debug", "refMap", ref2hash, "msg", "ref2hash map updated") + } + return nil + }) +} + +// semi-todo make shell implement http.FileSystem +// then we can reuse filepath.Walk and make a lot of other stuff simpler +var SkipDir = errors.Errorf("walk: skipping") + +type WalkFunc func(path string, info *shell.LsLink, err error) error + +func walk(path string, info *shell.LsLink, walkFn WalkFunc) error { + err := walkFn(path, info, nil) + if err != nil { + if info.Type == 1 && err == SkipDir { + return nil + } + return err + } + if info.Type != 1 { + return nil + } + list, err := ipfsShell.List(path) + if err != nil { + log.Log("msg", "walk list failed", "err", err) + + return walkFn(path, info, err) + } + for _, lnk := range list { + fname := filepath.Join(path, lnk.Name) + err = walk(fname, lnk, walkFn) + if err != nil { + if lnk.Type != 1 || err != SkipDir { + return err + } + } + } + return nil +} + +func Walk(root string, walkFn WalkFunc) error { + list, err := ipfsShell.List(root) + if err != nil { + log.Log("msg", "walk root failed", "err", err) + return walkFn(root, nil, err) + } + for _, l := range list { + fname := filepath.Join(root, l.Name) + if err := walk(fname, l, walkFn); err != nil { + return err + } + } + return nil +} diff --git a/git-remote-helper/main.go b/git-remote-helper/main.go new file mode 100644 index 0000000..268ae27 --- /dev/null +++ b/git-remote-helper/main.go @@ -0,0 +1,234 @@ +package main + +import ( + "bufio" + "fmt" + "io" + "os" + "path/filepath" + "reflect" + "strings" + + "github.com/capturealpha/gitblox/git-remote-gitblox/internal/path" + "github.com/cryptix/go/logging" + shell "github.com/ipfs/go-ipfs-api" + "github.com/pkg/errors" +) + +const usageMsg = `usage git-remote-gitblox [] +supports: + +* gitblox://ipfs/$hash/path.. +* gitblox:///ipfs/$hash/path.. + +` + +func usage() { + fmt.Fprint(os.Stderr, usageMsg) + os.Exit(2) +} + +var ( + ref2hash = make(map[string]string) + + ipfsShell = shell.NewShell("localhost:5001") + ipfsRepoPath string + thisGitRepo string + thisGitRemote string + log logging.Interface + check = logging.CheckFatal +) + +func logFatal(msg string) { + log.Log("event", "fatal", "msg", msg) + os.Exit(1) +} + +func main() { + // logging + logging.SetupLogging(nil) + log = logging.Logger("git-remote-gitblox") + + // env var and arguments + thisGitRepo = os.Getenv("GIT_DIR") + log.Log("GIT_DIR", thisGitRepo) + + if thisGitRepo == "" { + logFatal("could not get GIT_DIR env var") + } + if thisGitRepo == ".git" { + cwd, err := os.Getwd() + logging.CheckFatal(err) + thisGitRepo = filepath.Join(cwd, ".git") + } + + var u string // repo url + v := len(os.Args[1:]) + switch v { + case 2: + thisGitRemote = os.Args[1] + u = os.Args[2] + default: + logFatal(fmt.Sprintf("usage: unknown # of args: %d\n%v", v, os.Args[1:])) + } + + log.Log("thisGitRemote", thisGitRemote) + log.Log("u", u) + + // parse passed URL + for _, pref := range []string{"gitblox://ipfs/", "gitblox:///ipfs/"} { + if strings.HasPrefix(u, pref) { + u = "/ipfs/" + u[len(pref):] + } + } + log.Log("u", u) + parts := strings.Split(u, "/") + log.Log("parts", parts[0]) + p, err := path.ParsePath(u) + check(err) + + ipfsRepoPath = p.String() + log.Log("ipfsRepoPath", ipfsRepoPath) + + // interrupt / error handling + go func() { + check(interrupt()) + }() + + //refsCat, err := ipfsShell.Cat(filepath.Join(ipfsRepoPath, "info", "refs")) + ls, err := ipfsShell.List(filepath.Join(ipfsRepoPath, "info", "refs")) + if err != nil { + fmt.Fprintf(os.Stderr, "error: %s", err) + os.Exit(1) + } + if ls == nil { + fmt.Println("ls=nil") + os.Exit(1) + } + + for index, element := range ls { + log.Log("At index", index, "value is", element.Name) + } + + check(speakGit(os.Stdin, os.Stdout)) +} + +// speakGit acts like a git-remote-helper +// see this for more: https://www.kernel.org/pub/software/scm/git/docs/gitremote-helpers.html +func speakGit(r io.Reader, w io.Writer) error { + //debugLog := logging.Logger("git") + //r = debug.NewReadLogrus(debugLog, r) + //w = debug.NewWriteLogrus(debugLog, w) + scanner := bufio.NewScanner(r) + for scanner.Scan() { + text := scanner.Text() + log.Log("text", text) + switch { + + case text == "capabilities": + fmt.Fprintln(w, "fetch") + fmt.Fprintln(w, "push") + fmt.Fprintln(w, "") + + case strings.HasPrefix(text, "list"): + var ( + forPush = strings.Contains(text, "for-push") + err error + head string + ) + if err = listInfoRefs(forPush); err == nil { // try .git/info/refs first + if head, err = listHeadRef(); err != nil { + return err + } + } else { // alternativly iterate over the refs directory + if forPush { + log.Log("msg", "for-push: should be able to push to non existant.. TODO #2") + } + log.Log("err", err, "msg", "didn't find info/refs in repo, falling back...") + if err = listIterateRefs(forPush); err != nil { + return err + } + } + if len(ref2hash) == 0 { + return errors.New("did not find _any_ refs...") + } + // output + for ref, hash := range ref2hash { + if head == "" && strings.HasSuffix(ref, "master") { + // guessing head if it isnt set + head = hash + } + fmt.Fprintf(w, "%s %s\n", hash, ref) + } + fmt.Fprintf(w, "%s HEAD\n", head) + fmt.Fprintln(w) + + case strings.HasPrefix(text, "fetch "): + for scanner.Scan() { + fetchSplit := strings.Split(text, " ") + if len(fetchSplit) < 2 { + return errors.Errorf("malformed 'fetch' command. %q", text) + } + err := fetchObject(fetchSplit[1]) + if err == nil { + fmt.Fprintln(w) + continue + } + // TODO isNotExist(err) would be nice here + //log.Log("sha1", fetchSplit[1], "name", fetchSplit[2], "err", err, "msg", "fetchLooseObject failed, trying packed...") + + err = fetchPackedObject(fetchSplit[1]) + if err != nil { + return errors.Wrap(err, "fetchPackedObject() failed") + } + text = scanner.Text() + if text == "" { + break + } + } + fmt.Fprintln(w, "") + + case strings.HasPrefix(text, "push"): + for scanner.Scan() { + pushSplit := strings.Split(text, " ") + if len(pushSplit) < 2 { + return errors.Errorf("malformed 'push' command. %q", text) + } + srcDstSplit := strings.Split(pushSplit[1], ":") + if len(srcDstSplit) < 2 { + return errors.Errorf("malformed 'push' command. %q", text) + } + src, dst := srcDstSplit[0], srcDstSplit[1] + f := []interface{}{ + "src", src, + "dst", dst, + } + log.Log(append(f, "msg", "got push")) + if src == "" { + fmt.Fprintf(w, "error %s %s\n", dst, "delete remote dst: not supported yet - please open an issue on github") + } else { + if err := push(src, dst); err != nil { + fmt.Fprintf(w, "error %s %s\n", dst, err) + return err + } + fmt.Fprintln(w, "ok", dst) + } + text = scanner.Text() + if text == "" { + break + } + } + fmt.Fprintln(w, "") + + case text == "": + break + + default: + return errors.Errorf("Error: default git speak: %q", text) + } + } + if err := scanner.Err(); err != nil { + return errors.Wrap(err, "scanner.Err()") + } + return nil +} diff --git a/git-remote-helper/push.go b/git-remote-helper/push.go new file mode 100644 index 0000000..d9f56df --- /dev/null +++ b/git-remote-helper/push.go @@ -0,0 +1,117 @@ +package main + +import ( + "bytes" + "fmt" + "os/exec" + "path/filepath" + "strings" + + "github.com/pkg/errors" +) + +func push(src, dst string) error { + var force = strings.HasPrefix(src, "+") + if force { + src = src[1:] + } + var present []string + for _, h := range ref2hash { + present = append(present, h) + } + // also: track previously pushed branches in 2nd map and extend present with it + need2push, err := gitListObjects(src, present) + if err != nil { + return errors.Wrapf(err, "push: git list objects failed %q %v", src, present) + } + n := len(need2push) + type pair struct { + Sha1 string + MHash string + Err error + } + added := make(chan pair) + objHash2multi := make(map[string]string, n) + for _, sha1 := range need2push { + go func(sha1 string) { + r, err := gitFlattenObject(sha1) + if err != nil { + added <- pair{Err: errors.Wrapf(err, "gitFlattenObject failed")} + return + } + mhash, err := ipfsShell.Add(r) + if err != nil { + added <- pair{Err: errors.Wrapf(err, "shell.Add(%s) failed", sha1)} + return + } + added <- pair{Sha1: sha1, MHash: mhash} + }(sha1) + } + for n > 0 { + select { + // add timeout? + case p := <-added: + if p.Err != nil { + return p.Err + } + log.Log("sha1", p.Sha1, "mhash", p.MHash, "msg", "added") + objHash2multi[p.Sha1] = p.MHash + n-- + } + } + root, err := ipfsShell.ResolvePath(ipfsRepoPath) + if err != nil { + return errors.Wrapf(err, "resolvePath(%s) failed", ipfsRepoPath) + } + for sha1, mhash := range objHash2multi { + newRoot, err := ipfsShell.PatchLink(root, filepath.Join("objects", sha1[:2], sha1[2:]), mhash, true) + if err != nil { + return errors.Wrapf(err, "patchLink failed") + } + root = newRoot + log.Log("newRoot", newRoot, "sha1", sha1, "msg", "updated object") + } + srcSha1, err := gitRefHash(src) + if err != nil { + return errors.Wrapf(err, "gitRefHash(%s) failed", src) + } + h, ok := ref2hash[dst] + if !ok { + return errors.Errorf("writeRef: ref2hash entry missing: %s %+v", dst, ref2hash) + } + isFF := gitIsAncestor(h, srcSha1) + if isFF != nil && !force { + // TODO: print "non-fast-forward" to git + return errors.Errorf("non-fast-forward") + } + mhash, err := ipfsShell.Add(bytes.NewBufferString(fmt.Sprintf("%s\n", srcSha1))) + if err != nil { + return errors.Wrapf(err, "shell.Add(%s) failed", srcSha1) + } + root, err = ipfsShell.PatchLink(root, dst, mhash, true) + if err != nil { + // TODO:print "fetch first" to git + err = errors.Wrapf(err, "patchLink(%s) failed", ipfsRepoPath) + log.Log("err", err, "msg", "shell.PatchLink failed") + return errors.Errorf("fetch first") + } + log.Log("newRoot", root, "dst", dst, "hash", srcSha1, "msg", "updated ref") + // invalidate info/refs and HEAD(?) + // TODO: unclean: need to put other revs, too make a soft git update-server-info maybe + noInfoRefsHash, err := ipfsShell.Patch(root, "rm-link", "info/refs") + if err == nil { + log.Log("newRoot", noInfoRefsHash, "msg", "rm-link'ed info/refs") + root = noInfoRefsHash + } else { + // todo shell.IsNotExists() ? + log.Log("err", err, "msg", "shell.Patch rm-link info/refs failed - might be okay... TODO") + } + newRemoteURL := fmt.Sprintf("ipfs:///ipfs/%s", root) + updateRepoCMD := exec.Command("git", "remote", "set-url", thisGitRemote, newRemoteURL) + out, err := updateRepoCMD.CombinedOutput() + if err != nil { + return errors.Wrapf(err, "updating remote url failed\nOut:%s", string(out)) + } + log.Log("msg", "remote updated", "address", newRemoteURL) + return nil +}