Implement Cargo HTTP index (#24452)

This implements the HTTP index
[RFC](https://rust-lang.github.io/rfcs/2789-sparse-index.html) for Cargo
registries.

Currently this is a preview feature and you need to use the nightly of
`cargo`:

`cargo +nightly -Z sparse-registry update`

See https://github.com/rust-lang/cargo/issues/9069 for more information.

---------

Co-authored-by: Giteabot <teabot@gitea.io>
This commit is contained in:
KN4CK3R 2023-05-03 22:58:43 +02:00 committed by GitHub
parent 48e3e38ee0
commit 723598b803
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 136 additions and 38 deletions

View File

@ -119,6 +119,12 @@ func CommonRoutes(ctx gocontext.Context) *web.Route {
r.Get("/owners", cargo.ListOwners) r.Get("/owners", cargo.ListOwners)
}) })
}) })
r.Get("/config.json", cargo.RepositoryConfig)
r.Get("/1/{package}", cargo.EnumeratePackageVersions)
r.Get("/2/{package}", cargo.EnumeratePackageVersions)
// Use dummy placeholders because these parts are not of interest
r.Get("/3/{_}/{package}", cargo.EnumeratePackageVersions)
r.Get("/{_}/{__}/{package}", cargo.EnumeratePackageVersions)
}, reqPackageAccess(perm.AccessModeRead)) }, reqPackageAccess(perm.AccessModeRead))
r.Group("/chef", func() { r.Group("/chef", func() {
r.Group("/api/v1", func() { r.Group("/api/v1", func() {

View File

@ -4,6 +4,7 @@
package cargo package cargo
import ( import (
"errors"
"fmt" "fmt"
"net/http" "net/http"
"strconv" "strconv"
@ -45,6 +46,35 @@ func apiError(ctx *context.Context, status int, obj interface{}) {
}) })
} }
// https://rust-lang.github.io/rfcs/2789-sparse-index.html
func RepositoryConfig(ctx *context.Context) {
ctx.JSON(http.StatusOK, cargo_service.BuildConfig(ctx.Package.Owner))
}
func EnumeratePackageVersions(ctx *context.Context) {
p, err := packages_model.GetPackageByName(ctx, ctx.Package.Owner.ID, packages_model.TypeCargo, ctx.Params("package"))
if err != nil {
if errors.Is(err, util.ErrNotExist) {
apiError(ctx, http.StatusNotFound, err)
} else {
apiError(ctx, http.StatusInternalServerError, err)
}
return
}
b, err := cargo_service.BuildPackageIndex(ctx, p)
if err != nil {
apiError(ctx, http.StatusInternalServerError, err)
return
}
if b == nil {
apiError(ctx, http.StatusNotFound, nil)
return
}
ctx.PlainTextBytes(http.StatusOK, b.Bytes())
}
type SearchResult struct { type SearchResult struct {
Crates []*SearchResultCrate `json:"crates"` Crates []*SearchResultCrate `json:"crates"`
Meta SearchResultMeta `json:"meta"` Meta SearchResultMeta `json:"meta"`

View File

@ -137,21 +137,21 @@ type IndexVersionEntry struct {
Links string `json:"links,omitempty"` Links string `json:"links,omitempty"`
} }
func addOrUpdatePackageIndex(ctx context.Context, t *files_service.TemporaryUploadRepository, p *packages_model.Package) error { func BuildPackageIndex(ctx context.Context, p *packages_model.Package) (*bytes.Buffer, error) {
pvs, _, err := packages_model.SearchVersions(ctx, &packages_model.PackageSearchOptions{ pvs, _, err := packages_model.SearchVersions(ctx, &packages_model.PackageSearchOptions{
PackageID: p.ID, PackageID: p.ID,
Sort: packages_model.SortVersionAsc, Sort: packages_model.SortVersionAsc,
}) })
if err != nil { if err != nil {
return fmt.Errorf("SearchVersions[%s]: %w", p.Name, err) return nil, fmt.Errorf("SearchVersions[%s]: %w", p.Name, err)
} }
if len(pvs) == 0 { if len(pvs) == 0 {
return nil return nil, nil
} }
pds, err := packages_model.GetPackageDescriptors(ctx, pvs) pds, err := packages_model.GetPackageDescriptors(ctx, pvs)
if err != nil { if err != nil {
return fmt.Errorf("GetPackageDescriptors[%s]: %w", p.Name, err) return nil, fmt.Errorf("GetPackageDescriptors[%s]: %w", p.Name, err)
} }
var b bytes.Buffer var b bytes.Buffer
@ -179,14 +179,26 @@ func addOrUpdatePackageIndex(ctx context.Context, t *files_service.TemporaryUplo
Links: metadata.Links, Links: metadata.Links,
}) })
if err != nil { if err != nil {
return err return nil, err
} }
b.Write(entry) b.Write(entry)
b.WriteString("\n") b.WriteString("\n")
} }
return writeObjectToIndex(t, BuildPackagePath(pds[0].Package.LowerName), &b) return &b, nil
}
func addOrUpdatePackageIndex(ctx context.Context, t *files_service.TemporaryUploadRepository, p *packages_model.Package) error {
b, err := BuildPackageIndex(ctx, p)
if err != nil {
return err
}
if b == nil {
return nil
}
return writeObjectToIndex(t, BuildPackagePath(p.LowerName), b)
} }
func getOrCreateIndexRepository(ctx context.Context, doer, owner *user_model.User) (*repo_model.Repository, error) { func getOrCreateIndexRepository(ctx context.Context, doer, owner *user_model.User) (*repo_model.Repository, error) {
@ -212,6 +224,13 @@ type Config struct {
APIURL string `json:"api"` APIURL string `json:"api"`
} }
func BuildConfig(owner *user_model.User) *Config {
return &Config{
DownloadURL: setting.AppURL + "api/packages/" + owner.Name + "/cargo/api/v1/crates",
APIURL: setting.AppURL + "api/packages/" + owner.Name + "/cargo",
}
}
func createOrUpdateConfigFile(ctx context.Context, repo *repo_model.Repository, doer, owner *user_model.User) error { func createOrUpdateConfigFile(ctx context.Context, repo *repo_model.Repository, doer, owner *user_model.User) error {
return alterRepositoryContent( return alterRepositoryContent(
ctx, ctx,
@ -220,10 +239,7 @@ func createOrUpdateConfigFile(ctx context.Context, repo *repo_model.Repository,
"Initialize Cargo Config", "Initialize Cargo Config",
func(t *files_service.TemporaryUploadRepository) error { func(t *files_service.TemporaryUploadRepository) error {
var b bytes.Buffer var b bytes.Buffer
err := json.NewEncoder(&b).Encode(Config{ err := json.NewEncoder(&b).Encode(BuildConfig(owner))
DownloadURL: setting.AppURL + "api/packages/" + owner.Name + "/cargo/api/v1/crates",
APIURL: setting.AppURL + "api/packages/" + owner.Name + "/cargo",
})
if err != nil { if err != nil {
return err return err
} }

View File

@ -98,7 +98,7 @@ func testPackageCargo(t *testing.T, _ *neturl.URL) {
url := fmt.Sprintf("%s/api/v1/crates", root) url := fmt.Sprintf("%s/api/v1/crates", root)
t.Run("Index", func(t *testing.T) { t.Run("Index", func(t *testing.T) {
t.Run("Config", func(t *testing.T) { t.Run("Git/Config", func(t *testing.T) {
defer tests.PrintCurrentTest(t)() defer tests.PrintCurrentTest(t)()
content := readGitContent(t, cargo_service.ConfigFileName) content := readGitContent(t, cargo_service.ConfigFileName)
@ -110,6 +110,20 @@ func testPackageCargo(t *testing.T, _ *neturl.URL) {
assert.Equal(t, url, config.DownloadURL) assert.Equal(t, url, config.DownloadURL)
assert.Equal(t, root, config.APIURL) assert.Equal(t, root, config.APIURL)
}) })
t.Run("HTTP/Config", func(t *testing.T) {
defer tests.PrintCurrentTest(t)()
req := NewRequest(t, "GET", root+"/"+cargo_service.ConfigFileName)
resp := MakeRequest(t, req, http.StatusOK)
var config cargo_service.Config
err := json.Unmarshal(resp.Body.Bytes(), &config)
assert.NoError(t, err)
assert.Equal(t, url, config.DownloadURL)
assert.Equal(t, root, config.APIURL)
})
}) })
t.Run("Upload", func(t *testing.T) { t.Run("Upload", func(t *testing.T) {
@ -192,40 +206,72 @@ func testPackageCargo(t *testing.T, _ *neturl.URL) {
MakeRequest(t, req, http.StatusConflict) MakeRequest(t, req, http.StatusConflict)
t.Run("Index", func(t *testing.T) { t.Run("Index", func(t *testing.T) {
t.Run("Entry", func(t *testing.T) { t.Run("Git", func(t *testing.T) {
defer tests.PrintCurrentTest(t)() t.Run("Entry", func(t *testing.T) {
defer tests.PrintCurrentTest(t)()
content := readGitContent(t, cargo_service.BuildPackagePath(packageName)) content := readGitContent(t, cargo_service.BuildPackagePath(packageName))
var entry cargo_service.IndexVersionEntry var entry cargo_service.IndexVersionEntry
err := json.Unmarshal([]byte(content), &entry) err := json.Unmarshal([]byte(content), &entry)
assert.NoError(t, err) assert.NoError(t, err)
assert.Equal(t, packageName, entry.Name) assert.Equal(t, packageName, entry.Name)
assert.Equal(t, packageVersion, entry.Version) assert.Equal(t, packageVersion, entry.Version)
assert.Equal(t, pb.HashSHA256, entry.FileChecksum) assert.Equal(t, pb.HashSHA256, entry.FileChecksum)
assert.False(t, entry.Yanked) assert.False(t, entry.Yanked)
assert.Len(t, entry.Dependencies, 1) assert.Len(t, entry.Dependencies, 1)
dep := entry.Dependencies[0] dep := entry.Dependencies[0]
assert.Equal(t, "dep", dep.Name) assert.Equal(t, "dep", dep.Name)
assert.Equal(t, "1.0", dep.Req) assert.Equal(t, "1.0", dep.Req)
assert.Equal(t, "normal", dep.Kind) assert.Equal(t, "normal", dep.Kind)
assert.True(t, dep.DefaultFeatures) assert.True(t, dep.DefaultFeatures)
assert.Empty(t, dep.Features) assert.Empty(t, dep.Features)
assert.False(t, dep.Optional) assert.False(t, dep.Optional)
assert.Nil(t, dep.Target) assert.Nil(t, dep.Target)
assert.NotNil(t, dep.Registry) assert.NotNil(t, dep.Registry)
assert.Equal(t, "https://gitea.io/user/_cargo-index", *dep.Registry) assert.Equal(t, "https://gitea.io/user/_cargo-index", *dep.Registry)
assert.Nil(t, dep.Package) assert.Nil(t, dep.Package)
})
t.Run("Rebuild", func(t *testing.T) {
defer tests.PrintCurrentTest(t)()
err := cargo_service.RebuildIndex(db.DefaultContext, user, user)
assert.NoError(t, err)
_ = readGitContent(t, cargo_service.BuildPackagePath(packageName))
})
}) })
t.Run("Rebuild", func(t *testing.T) { t.Run("HTTP", func(t *testing.T) {
defer tests.PrintCurrentTest(t)() t.Run("Entry", func(t *testing.T) {
defer tests.PrintCurrentTest(t)()
err := cargo_service.RebuildIndex(db.DefaultContext, user, user) req := NewRequest(t, "GET", root+"/"+cargo_service.BuildPackagePath(packageName))
assert.NoError(t, err) resp := MakeRequest(t, req, http.StatusOK)
_ = readGitContent(t, cargo_service.BuildPackagePath(packageName)) var entry cargo_service.IndexVersionEntry
err := json.Unmarshal(resp.Body.Bytes(), &entry)
assert.NoError(t, err)
assert.Equal(t, packageName, entry.Name)
assert.Equal(t, packageVersion, entry.Version)
assert.Equal(t, pb.HashSHA256, entry.FileChecksum)
assert.False(t, entry.Yanked)
assert.Len(t, entry.Dependencies, 1)
dep := entry.Dependencies[0]
assert.Equal(t, "dep", dep.Name)
assert.Equal(t, "1.0", dep.Req)
assert.Equal(t, "normal", dep.Kind)
assert.True(t, dep.DefaultFeatures)
assert.Empty(t, dep.Features)
assert.False(t, dep.Optional)
assert.Nil(t, dep.Target)
assert.NotNil(t, dep.Registry)
assert.Equal(t, "https://gitea.io/user/_cargo-index", *dep.Registry)
assert.Nil(t, dep.Package)
})
}) })
}) })
}) })