Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 3 additions & 1 deletion docs/llms.txt

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

4 changes: 3 additions & 1 deletion docs/python.md
Original file line number Diff line number Diff line change
Expand Up @@ -544,7 +544,9 @@ These types can be used directly as input parameter types and output return type

`cog.Path` is used to get files in and out of models. It represents a _path to a file on disk_.

`cog.Path` is a subclass of Python's [`pathlib.Path`](https://docs.python.org/3/library/pathlib.html#basic-use) and can be used as a drop-in replacement. Any `os.PathLike` subclass is also accepted as an input type and treated as `cog.Path`.
File inputs must be `cog.Path` (`from cog import Path`). Clients send a URL; Cog downloads it to a temp file before `run()`. `from pathlib import Path` on an input fails `cog build`, because that download only happens for `cog.Path`.

`cog.Path` is a subclass of Python's [`pathlib.Path`](https://docs.python.org/3/library/pathlib.html#basic-use). A return type of `pathlib.Path` still works: Cog uploads any `os.PathLike`.

For models that return a `cog.Path` object, the output returned by Cog's built-in HTTP server will be a URL.

Expand Down
18 changes: 17 additions & 1 deletion pkg/schema/errors.go
Original file line number Diff line number Diff line change
@@ -1,6 +1,9 @@
package schema

import "fmt"
import (
"fmt"
"strings"
)

// SchemaError represents errors during schema generation.
type SchemaError struct {
Expand Down Expand Up @@ -112,6 +115,19 @@ func errUnresolvableImportedType(name, module string) error {
}
}

func errNotCogFileLike(localName, module, original string) error {
msg := fmt.Sprintf(
"%s is %s.%s, not cog.%s. Use `from cog import %s` for file inputs",
localName, module, original, original, original)
if !strings.Contains(localName, ".") {
msg += fmt.Sprintf(". If you also need %s.%s, import it under a different name", module, original)
}
return &SchemaError{
Kind: ErrUnsupportedType,
Message: msg,
}
}

func errUnresolvableType(name string) error {
return &SchemaError{
Kind: ErrUnresolvableType,
Expand Down
131 changes: 131 additions & 0 deletions pkg/schema/python/models.go
Original file line number Diff line number Diff line change
Expand Up @@ -144,6 +144,7 @@ func (ctx *modelParseContext) loadModelsFromModule(sourceDir, module string) sch
if pyPath == "" {
return nil
}
pyPath = existingPythonFile(sourceDir, pyPath)
cacheKey := filepath.Clean(pyPath)
if summary, ok := ctx.loadedModules[cacheKey]; ok {
return summary.Models
Expand Down Expand Up @@ -260,6 +261,110 @@ func nestedImportModule(module string, original string) string {
return module + "." + original
}

// followReexportedFileLikes rewrites Path/File/Secret imports that came from a
// local module so they point at the original cog or pathlib binding.
//
// # types.py
// from pathlib import Path
// # predict.py
// from .types import Path
//
// becomes pathlib.Path, which inputs then reject. Unresolved relative imports
// (no file on disk) are left as file URIs, same as a missing local BaseModel.
func followReexportedFileLikes(imports *schema.ImportContext, loaded map[string]ModuleSummary, sourcePath string) {
if imports == nil || loaded == nil || len(loaded) == 0 {
return
}
type rewrite struct {
local string
entry schema.ImportEntry
}
var rewrites []rewrite
imports.Names.Entries(func(localName string, entry schema.ImportEntry) {
resolved := followFileLikeOrigin(entry, loaded, sourcePath, 0)
if resolved == entry {
return
}
if resolved.Original != "Path" && resolved.Original != "File" && resolved.Original != "Secret" {
return
}
if resolved.Module != "cog" && !strings.HasPrefix(resolved.Module, "cog.") &&
resolved.Module != "pathlib" && !strings.HasPrefix(resolved.Module, "pathlib.") {
return
}
rewrites = append(rewrites, rewrite{local: localName, entry: resolved})
})
for _, r := range rewrites {
imports.Names.Set(r.local, r.entry)
}
recordImportedModuleFileLikes(imports, loaded, sourcePath)
}

// recordImportedModuleFileLikes records Path/File/Secret on `import helpers`
// so `helpers.Path` follows the same pathlib/cog origin as `from helpers import Path`.
func recordImportedModuleFileLikes(imports *schema.ImportContext, loaded map[string]ModuleSummary, sourcePath string) {
if imports.ModuleAttrs == nil {
imports.ModuleAttrs = map[string]map[string]schema.ImportEntry{}
}
imports.Names.Entries(func(localName string, entry schema.ImportEntry) {
// parseImport stores Original == Module for `import foo` and `import foo as bar`.
if entry.Original != entry.Module {
return
}
if isKnownExternalModule(entry.Module) {
return
}
pyPath := moduleToFilePath(entry.Module, sourcePath)
summary, ok := lookupLoaded(loaded, pyPath)
if !ok || summary.Imports == nil {
return
}
attrs := map[string]schema.ImportEntry{}
summary.Imports.Names.Entries(func(attr string, inner schema.ImportEntry) {
resolved := followFileLikeOrigin(inner, loaded, summary.SourcePath, 0)
if resolved.Original != "Path" && resolved.Original != "File" && resolved.Original != "Secret" {
return
}
if resolved.Module != "cog" && !strings.HasPrefix(resolved.Module, "cog.") &&
resolved.Module != "pathlib" && !strings.HasPrefix(resolved.Module, "pathlib.") {
return
}
attrs[attr] = resolved
})
if len(attrs) > 0 {
imports.ModuleAttrs[localName] = attrs
}
})
}

func followFileLikeOrigin(entry schema.ImportEntry, loaded map[string]ModuleSummary, sourcePath string, depth int) schema.ImportEntry {
if depth > 8 {
return entry
}
if entry.Module == "cog" || strings.HasPrefix(entry.Module, "cog.") {
return entry
}
if entry.Module == "pathlib" || strings.HasPrefix(entry.Module, "pathlib.") {
return entry
}
if isKnownExternalModule(entry.Module) {
return entry
}
pyPath := moduleToFilePath(entry.Module, sourcePath)
if pyPath == "" {
return entry
}
summary, ok := lookupLoaded(loaded, pyPath)
if !ok || summary.Imports == nil {
return entry
}
next, ok := summary.Imports.Names.Get(entry.Original)
if !ok {
return entry
}
return followFileLikeOrigin(next, loaded, summary.SourcePath, depth+1)
}

func refreshLoadedModuleAliases(loadedModules map[string]ModuleSummary) {
for _, summary := range loadedModules {
if summary.Imports == nil || summary.Models == nil {
Expand All @@ -279,6 +384,32 @@ func refreshLoadedModuleAliases(loadedModules map[string]ModuleSummary) {
}
}

func lookupLoaded(loaded map[string]ModuleSummary, pyPath string) (ModuleSummary, bool) {
if pyPath == "" {
return ModuleSummary{}, false
}
if summary, ok := loaded[filepath.Clean(pyPath)]; ok {
return summary, true
}
initPath := filepath.Join(strings.TrimSuffix(pyPath, ".py"), "__init__.py")
summary, ok := loaded[filepath.Clean(initPath)]
return summary, ok
}

func existingPythonFile(sourceDir, pyPath string) string {
if sourceDir == "" || pyPath == "" {
return pyPath
}
if _, err := os.Stat(filepath.Join(sourceDir, pyPath)); err == nil {
return pyPath
}
initRel := filepath.Join(strings.TrimSuffix(pyPath, ".py"), "__init__.py")
if _, err := os.Stat(filepath.Join(sourceDir, initRel)); err == nil {
return initRel
}
return pyPath
}

// moduleToFilePath converts a Python module path to a relative .py file path.
//
// ".types", "pkg/predict.py" → "pkg/types.py"
Expand Down
1 change: 1 addition & 0 deletions pkg/schema/python/parser.go
Original file line number Diff line number Diff line change
Expand Up @@ -156,6 +156,7 @@ func resolveImportedModelsPhase(state *ParseState) error {
state.ModelCtx.resolvedModels = state.Models
setDiscoveredModels(state.Models, collectModelClasses(state.Root, state.Options.Source, state.ModelCtx))
}
followReexportedFileLikes(state.Imports, state.LoadedModules, state.Options.SourcePath)
return nil
}

Expand Down
Loading