Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
29 changes: 17 additions & 12 deletions api/process/controller/link.go
Original file line number Diff line number Diff line change
Expand Up @@ -54,18 +54,18 @@ func NewLinkController(svc *service.LinkService) *LinkController {
svc: svc,
proxy: &httputil.ReverseProxy{
Transport: svc.Registry().Transport(),
// The target was set on the request by jupyter; Rewrite only keeps the
// The target was set on the request by linkspan; Rewrite only keeps the
// client's forwarding headers out.
Rewrite: func(out *httputil.ProxyRequest) {
out.Out.URL.Scheme = "http"
out.Out.URL.Host = out.In.URL.Host
out.Out.Host = out.In.Host
},
ErrorHandler: func(w http.ResponseWriter, r *http.Request, err error) {
slog.Warn("Jupyter proxy could not reach the process's server", "path", r.URL.Path, "error", err)
slog.Warn("Linkspan proxy could not reach the process's job", "path", r.URL.Path, "error", err)
httpx.WriteJSON(w, http.StatusBadGateway, map[string]string{
"error": "Bad Gateway",
"message": "The process's Jupyter Server could not be reached: its job is not linked to Airavata, or its server has not started",
"message": "The process's Linkspan could not be reached: its job is not linked to Airavata, or the server has not started",
})
},
},
Expand All @@ -76,14 +76,14 @@ func NewLinkController(svc *service.LinkService) *LinkController {
func (h *LinkController) Register(mux *http.ServeMux) {
mux.HandleFunc("GET /api/v1/processes/{processId}/link", h.link)
mux.HandleFunc("GET /api/v1/processes/{processId}/jupyter-server", h.jupyterServer)
mux.HandleFunc("/api/v1/processes/{processId}/jupyter/", h.jupyter)
mux.HandleFunc("/api/v1/processes/{processId}/linkspan/", h.proxyToLinkspan)
}

// link admits the WebSocket a process's Linkspan dials and holds it as the process's
// link until it closes.
func (h *LinkController) link(w http.ResponseWriter, r *http.Request) {
processID := r.PathValue("processId")
if !h.svc.Authenticate(r.Context(), processID, link.OfferedToken(r)) {
if !h.svc.AuthenticateAndMarkRunning(r.Context(), processID, link.OfferedToken(r)) {
httpx.WriteError(w, r, httpx.Unauthorized("The link token is not the one issued to process %s", processID))
return
}
Expand Down Expand Up @@ -112,30 +112,35 @@ func (h *LinkController) jupyterServer(w http.ResponseWriter, r *http.Request) {
httpx.WriteJSON(w, http.StatusOK, resp)
}

// jupyter proxies the Jupyter Server API, kernel WebSockets included, to the server
// the process's job runs, over its link. The token is left on the request: the
// Jupyter Server checks it again.
func (h *LinkController) jupyter(w http.ResponseWriter, r *http.Request) {
// proxyToLinkspan proxies over the process's link to Linkspan's API, or to the port itself under
// forward/{port}/. The token stays on the request: the Jupyter Server checks it again.
func (h *LinkController) proxyToLinkspan(w http.ResponseWriter, r *http.Request) {
processID := r.PathValue("processId")
token := r.URL.Query().Get("token")
if scheme, value, found := strings.Cut(r.Header.Get("Authorization"), " "); found && strings.EqualFold(scheme, "token") {
token = strings.TrimSpace(value)
}
port, ok := h.svc.JupyterPort(r.Context(), processID, token)
port, ok := h.svc.LinkspanPort(r.Context(), processID, token)
if !ok {
httpx.WriteError(w, r, httpx.Unauthorized("The Jupyter token is not the one issued to process %s", processID))
return
}

prefix := "/api/v1/processes/" + processID + "/jupyter"
prefix := "/api/v1/processes/" + processID + "/linkspan"
out := r.Clone(r.Context())
out.URL.Path = strings.TrimPrefix(r.URL.Path, prefix)
out.URL.RawPath = ""
if out.URL.Path == "" {
out.URL.Path = "/"
}
// A bare forward/{port} is Linkspan's own TCP stream; leave it.
if rawPort, path, ok := strings.Cut(strings.TrimPrefix(out.URL.Path, "/api/v1/forward/"), "/"); ok {
if forwarded, err := strconv.ParseUint(rawPort, 10, 16); err == nil {
port, out.URL.Path = uint16(forwarded), "/"+path
}
}
out.URL.Host = link.Host(processID, port)
// The Jupyter Server sees itself addressed on its own loopback port.
// The server sees itself addressed on its own loopback port.
out.Host = "127.0.0.1:" + strconv.Itoa(int(port))
h.proxy.ServeHTTP(w, out)
}
31 changes: 21 additions & 10 deletions api/process/service/link.go
Original file line number Diff line number Diff line change
Expand Up @@ -28,13 +28,16 @@ import (
"encoding/base64"
"encoding/hex"
"fmt"
"log/slog"
"net/url"
"strconv"
"strings"
"time"

"github.com/apache/airavata/internal/auth"
"github.com/apache/airavata/internal/httpx"
"github.com/apache/airavata/internal/link"
"github.com/apache/airavata/internal/ptr"

dto "github.com/apache/airavata/api/process/dto"
model "github.com/apache/airavata/api/process/model"
Expand All @@ -48,14 +51,15 @@ import (
type LinkService struct {
links *repository.LinkRepository
processes *repository.ProcessRepository
statuses *StatusService
registry *link.Registry
publicURL string
}

// NewLinkService returns a link service. publicURL is where jobs reach this server,
// http(s)://host[:port]; the link URL is its ws(s) form.
func NewLinkService(links *repository.LinkRepository, processes *repository.ProcessRepository, registry *link.Registry, publicURL string) *LinkService {
return &LinkService{links: links, processes: processes, registry: registry, publicURL: strings.TrimRight(publicURL, "/")}
func NewLinkService(links *repository.LinkRepository, processes *repository.ProcessRepository, statuses *StatusService, registry *link.Registry, publicURL string) *LinkService {
return &LinkService{links: links, processes: processes, statuses: statuses, registry: registry, publicURL: strings.TrimRight(publicURL, "/")}
}

// Registry is the live links this service admits.
Expand Down Expand Up @@ -107,16 +111,23 @@ func (s *LinkService) Issue(ctx context.Context, processID string) (*link.Issued
return &link.Issued{URL: linkURL, LinkToken: linkToken, JupyterToken: jupyterToken, ControlPort: control, JupyterPort: jupyter}, nil
}

// Authenticate reports whether token is the link token last issued to the process.
func (s *LinkService) Authenticate(ctx context.Context, processID, token string) bool {
// AuthenticateAndMarkRunning reports whether token is the link token last issued to the process.
// A match records a SUBMITTED process as RUNNING, without waiting on Slurm's begin mail.
func (s *LinkService) AuthenticateAndMarkRunning(ctx context.Context, processID, token string) bool {
if token == "" {
return false
}
found, err := s.links.FindByProcessID(ctx, processID)
if err != nil {
if err != nil || subtle.ConstantTimeCompare([]byte(digest(token)), []byte(found.LinkTokenHash)) != 1 {
return false
}
return subtle.ConstantTimeCompare([]byte(digest(token)), []byte(found.LinkTokenHash)) == 1
if statuses, err := s.statuses.ListForProcess(ctx, processID); err == nil && len(statuses) > 0 &&
ptr.From(statuses[len(statuses)-1].Status) == model.ProcessStatusTypeSubmitted {
if _, err := s.statuses.Record(ctx, processID, model.ProcessStatusTypeRunning, nil); err != nil {
slog.Warn("Failed to record a linking process as running", "processId", processID, "error", err)
}
}
return true
}

// JupyterServer returns the caller's access to their process's Jupyter Server. Only
Expand All @@ -140,22 +151,22 @@ func (s *LinkService) JupyterServer(ctx context.Context, processID string) (*dto
return &dto.JupyterServerResponse{
ProcessID: processID,
Linked: s.registry.Linked(processID),
BaseURL: "/api/v1/processes/" + url.PathEscape(processID) + "/jupyter/",
BaseURL: "/api/v1/processes/" + url.PathEscape(processID) + "/linkspan/api/v1/forward/" + strconv.Itoa(int(found.JupyterPort)) + "/",
Token: found.JupyterToken,
}, nil
}

// JupyterPort returns the port of the process's Jupyter Server when token is its
// LinkspanPort returns the port of the process's Linkspan API when token is its
// Jupyter token, and false otherwise.
func (s *LinkService) JupyterPort(ctx context.Context, processID, token string) (uint16, bool) {
func (s *LinkService) LinkspanPort(ctx context.Context, processID, token string) (uint16, bool) {
if token == "" {
return 0, false
}
found, err := s.links.FindByProcessID(ctx, processID)
if err != nil || subtle.ConstantTimeCompare([]byte(token), []byte(found.JupyterToken)) != 1 {
return 0, false
}
return uint16(found.JupyterPort), true
return uint16(found.ControlPort), true
}

func newSecret() (string, error) {
Expand Down
2 changes: 1 addition & 1 deletion dashboard/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,7 @@ src/components/NotebookView.tsx, NotebookFiles.tsx
the Notebooks page: a file browser and a @datalayer/jupyter-react
notebook, either on an in-browser Pyodide kernel with files in
the browser's IndexedDB, or on the Jupyter Server of a process's
Linkspan job, through the API's /processes/{id}/jupyter/ proxy
Linkspan job, through the API's /processes/{id}/linkspan/ proxy
```

The build uses webpack (`next build --webpack`), not Turbopack: the JupyterLab packages
Expand Down
8 changes: 4 additions & 4 deletions docs/notebooks.md
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ The job reaches Airavata the way a [CyberShuttle](https://github.com/cyber-shutt

```
browser ──HTTPS──▶ Airavata ──link (WebSocket, yamux)──▶ Linkspan ──loopback──▶ Jupyter Server
/api/v1/processes/{id}/jupyter/ dialled from the job on the compute node
/api/v1/processes/{id}/linkspan/ dialled from the job on the compute node
```

## What every batch job is given
Expand Down Expand Up @@ -54,17 +54,17 @@ This needs Linkspan 0.22.0 or newer, the first release with the link transport,

On the Notebooks page, **Run notebooks** lists the signed-in user's processes that are submitted or running. Choosing one waits for its Linkspan to link, then opens the job's Jupyter Server: the file browser shows the job's files, and notebooks run on a `python3` kernel on the compute node. The kernel is shut down when the page is left or another source is chosen. Choosing **In this browser** goes back to Pyodide and the notebooks kept in the browser.

If Airavata restarts, or the link drops, Linkspan redials on its own and the page picks the server up again.
If Airavata restarts, or the link drops, Linkspan redials on its own and the page picks the server up again. A `SUBMITTED` process is recorded `RUNNING` when its Linkspan dials, without waiting for Slurm's begin mail.

## API

| Route | Authentication | Purpose |
|---|---|---|
| `GET /api/v1/processes/{processId}/link` | The link token, as the WebSocket subprotocol `link.<token>` | Linkspan's link. Airavata selects the subprotocol `cybershuttle.v1`. |
| `GET /api/v1/processes/{processId}/jupyter-server` | Bearer token of the process owner | `{processId, linked, baseUrl, token}`: where to reach the Jupyter Server and its token |
| `/api/v1/processes/{processId}/jupyter/…` | The Jupyter token, as `Authorization: token <token>` or `?token=` | Proxy to the job's Jupyter Server, including kernel WebSockets. `502` while the job is not linked. |
| `/api/v1/processes/{processId}/linkspan/…` | The Jupyter token, as `Authorization: token <token>` or `?token=` | Proxy to [Linkspan's API](https://github.com/cyber-shuttle/linkspan#http-api) in the job, WebSockets included. `api/v1/forward/{port}/…` goes to that port; `baseUrl` uses it. `502` while unlinked. |

Only the owner of a process can read its Jupyter token, because the token runs code as the owner's cluster account. The proxy compares the token before forwarding, and the Jupyter Server checks it again. Airavata's bearer tokens are never sent to the job. Other users on a shared node can reach the job's loopback ports, but they still need the Jupyter token to use the server ([Linkspan's security model](https://github.com/cyber-shuttle/linkspan/blob/main/SECURITY.md#security-model)).
Only the owner of a process can read its Jupyter token, because the token runs code as the owner's cluster account. The proxy compares the token before forwarding. The Jupyter Server checks it again; Linkspan's own API does not. Airavata's bearer tokens are never sent to the job. Other users on a shared node can reach the job's loopback ports, but they still need the Jupyter token to use the Jupyter Server ([Linkspan's security model](https://github.com/cyber-shuttle/linkspan/blob/main/SECURITY.md#security-model)).

The link protocol is Linkspan's, as frozen in its [compatibility document](https://github.com/cyber-shuttle/linkspan/blob/main/docs/COMPATIBILITY.md): yamux runs over binary WebSocket frames, with Airavata as the yamux client. For each stream Airavata writes the target port as two big-endian bytes, and Linkspan answers `1` and carries the stream if one of its own servers listens there, or answers `0`.

2 changes: 1 addition & 1 deletion internal/app/app.go
Original file line number Diff line number Diff line change
Expand Up @@ -243,7 +243,7 @@ func NewServices(cfg config.Config, db *gorm.DB, repos *Repositories, workflowBa
virtualDataDirectorySharingSvc := datasvc.NewVirtualDataDirectorySharingService(db, virtualDirs, shares, groups, users, groupMembers)
sshCredentialSvc := credentialssvc.NewSSHCredentialService(sshCredentials, users, clusterConfigs, storages)

linkSvc := processsvc.NewLinkService(repos.Links, processes, link.NewRegistry(), cfg.PublicURL)
linkSvc := processsvc.NewLinkService(repos.Links, processes, statusSvc, link.NewRegistry(), cfg.PublicURL)

executionEngine := orchestration.NewExecutionEngine(stagingTasks, submissionTasks,
monitoringTasks, products, storages, clusterConfigs, processes, deployments, templates, batchStatus, statuses, repos.TaskStatuses, processWorkflows, workflowBackend, sshContext, linkSvc)
Expand Down
3 changes: 3 additions & 0 deletions internal/mcp/client/client.gen.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

13 changes: 10 additions & 3 deletions internal/server/link_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -140,7 +140,7 @@ func TestLinkCarriesTheJupyterServerToItsOwner(t *testing.T) {
wsBase := "ws" + strings.TrimPrefix(api.URL, "http")
linkURL := wsBase + "/api/v1/processes/" + processID + "/link"
jupyter := fakeJupyter(t)
proxy := api.URL + "/api/v1/processes/" + processID + "/jupyter"
proxy := api.URL + "/api/v1/processes/" + processID + "/linkspan/api/v1/forward/" + itoa(jupyterPort)

get := func(path, authorization string) (*http.Response, map[string]string) {
req, _ := http.NewRequest(http.MethodGet, proxy+path, nil)
Expand All @@ -157,6 +157,10 @@ func TestLinkCarriesTheJupyterServerToItsOwner(t *testing.T) {
return resp, body
}

if _, err := h.svcs.ProcessStatus.Record(ctx, processID, "SUBMITTED", nil); err != nil {
t.Fatal(err)
}

// Before Linkspan links, the proxy has nowhere to go.
if resp, _ := get("/api/status", "token "+issued.JupyterToken); resp.StatusCode != http.StatusBadGateway {
t.Errorf("unlinked proxy: status = %d, want 502", resp.StatusCode)
Expand Down Expand Up @@ -184,9 +188,12 @@ func TestLinkCarriesTheJupyterServerToItsOwner(t *testing.T) {
time.Sleep(20 * time.Millisecond)
}
if access["linked"] != true || access["token"] != issued.JupyterToken ||
access["baseUrl"] != "/api/v1/processes/"+processID+"/jupyter/" {
access["baseUrl"] != "/api/v1/processes/"+processID+"/linkspan/api/v1/forward/"+itoa(jupyterPort)+"/" {
t.Fatalf("jupyter-server = %v", access)
}
if statuses := h.list("/api/v1/processes/"+processID+"/statuses", tokenAlice); statuses[len(statuses)-1]["status"] != "RUNNING" {
t.Errorf("statuses after link = %v, want the submitted process recorded RUNNING", statuses)
}

// The proxy wants the Jupyter token, not a bearer token.
if resp, _ := get("/api/status", ""); resp.StatusCode != http.StatusUnauthorized {
Expand All @@ -205,7 +212,7 @@ func TestLinkCarriesTheJupyterServerToItsOwner(t *testing.T) {
}

// Kernel channels are WebSockets with the token in the query.
wsURL := wsBase + "/api/v1/processes/" + processID + "/jupyter/api/kernels/k1/channels?token=" + issued.JupyterToken
wsURL := "ws" + strings.TrimPrefix(proxy, "http") + "/api/kernels/k1/channels?token=" + issued.JupyterToken
kernel, wsResp, err := websocket.Dial(ctx, wsURL, nil)
if wsResp != nil && wsResp.Body != nil {
wsResp.Body.Close()
Expand Down
Loading