diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f24301a..463e647 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -38,9 +38,9 @@ jobs: - run: npm run check:markers gaffer: - name: Gaffer host and panel consent (1.7.6.0, private profile) + name: Gaffer host, programs and panel consent (1.7.6.0, private profile) runs-on: windows-latest - timeout-minutes: 15 + timeout-minutes: 25 steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: @@ -57,11 +57,11 @@ jobs: Invoke-WebRequest 'https://github.com/GafferHQ/gaffer/releases/download/1.7.6.0/gaffer-1.7.6.0-windows.zip' -OutFile $archive if ((Get-FileHash $archive -Algorithm SHA256).Hash -ne '1B6C03F768E6FDFDE686A2AF256E59A313E1BFC64643371FA587DAC24AC69FC0') { throw 'Gaffer checksum mismatch' } Expand-Archive $archive -DestinationPath $env:RUNNER_TEMP - - name: Exercise the real graph, authenticated listener and offscreen panel + - name: Exercise OpenCode Link, the fixed programs and the offscreen panel in the real Gaffer shell: pwsh run: | $env:GAFFER_TEST_BIN = Join-Path $env:RUNNER_TEMP 'gaffer-1.7.6.0-windows/bin/gaffer.cmd' - node --test integrations/gaffer/test/real-host.test.ts + node --test --test-concurrency=1 integrations/gaffer/test/real-host.test.ts integrations/gaffer/test/gaffer.test.ts blender-native: name: Blender binding contract (OpenCode 2.2.5, private service) diff --git a/catalog.json b/catalog.json index e5952d4..90b8276 100644 --- a/catalog.json +++ b/catalog.json @@ -40,17 +40,17 @@ { "id": "gaffer", "package": "@neuralnomads/opencode-gaffer", - "range": "^0.2", + "range": "^0.3", "displayName": "OpenCode for Gaffer", "family": "native", "plugin": "neuralnomads.gaffer", "rpc": "neuralnomads.gaffer", "summary": { - "en": "Inspect and edit the open Gaffer graph through an authenticated host, from any OpenCode client or the in-app panel.", - "fr": "Inspecter et modifier le graphe Gaffer ouvert via un hôte authentifié, depuis tout client OpenCode ou le panneau intégré." + "en": "Read, change and check the scripts open in Gaffer from any OpenCode conversation, with an optional panel inside Gaffer.", + "fr": "Lire, modifier et vérifier les scripts ouverts dans Gaffer depuis toute conversation OpenCode, avec un panneau facultatif dans Gaffer." }, "app": { "name": "Gaffer", "versions": ">=1.6" }, - "host": { "protocol": 1, "range": "^0.2.0" }, + "host": { "protocol": 1, "range": "^0.3.0" }, "docs": "https://github.com/NeuralNomadsAI/opencode-integrations/blob/main/integrations/gaffer/plugin/README.md", "status": "preview", "sources": { "git": ["github:NeuralNomadsAI/opencode-integrations#main::path:integrations/gaffer/plugin"] } diff --git a/integrations/gaffer/host/manifest.json b/integrations/gaffer/host/manifest.json index a56c79a..9992a34 100644 --- a/integrations/gaffer/host/manifest.json +++ b/integrations/gaffer/host/manifest.json @@ -1 +1 @@ -{ "name": "OpenCode Link for Gaffer", "version": "0.2.0", "protocol": 1, "license": "MIT" } +{ "name": "OpenCode Link for Gaffer", "version": "0.3.0", "protocol": 1, "license": "MIT" } diff --git a/integrations/gaffer/host/python/GafferOpenCode/Advertiser.py b/integrations/gaffer/host/python/GafferOpenCode/Advertiser.py new file mode 100644 index 0000000..ce50907 --- /dev/null +++ b/integrations/gaffer/host/python/GafferOpenCode/Advertiser.py @@ -0,0 +1,191 @@ +"""Offers this Gaffer's open scripts to the discovered OpenCode services (standard library only). + +Runs on its own thread: discovery and RPC calls never block Gaffer's main thread. One offer per open script +(labelled with its file name), renewed every 5 seconds straight at the services that took it; a script closed is +withdrawn, and an offer whose renewals stop expires within 15 seconds anyway. Results carry service file names, +labels and outcomes only, never the daemon password nor the host token. +""" + +import os +import re +import secrets +import threading +import time + +RPC_ID = "neuralnomads.gaffer" +PROTOCOL = 1 +RENEW_SECONDS = 5.0 +# One renewal may wait this long for a busy service: an offer lasts 15 s, renewed every 5 s. +RENEW_TIMEOUT = 8.0 +# Renewals go straight to the services that took the offers; every this many renewals (or after a failure) the +# services are discovered again, which finds a service started since. +DISCOVER_EVERY = 12 + + +def _launchKey() : + + """A Gaffer started by OpenCode offers its first script under the key OpenCode chose.""" + + key = os.environ.get( "OPENCODE_GAFFER_OFFER_KEY", "" ) + return key if re.fullmatch( r"gaffer-launch-[0-9a-f]{32}", key ) else None + + +# Stable for the life of the Gaffer process; each script's key adds its number (the first script: none). +OFFER_KEY = _launchKey() or "gaffer-{}-{}".format( os.getpid(), secrets.token_hex( 8 ) ) + + +def offerKey( number ) : + + return OFFER_KEY if number == 0 else "{}.{}".format( OFFER_KEY, number ) + + +class Advertiser : + + """`snapshot()` returns ( port, token, [ ( script_handle, label, offer_key ), ... ] ), or a None token while not + ready.""" + + def __init__( self, snapshot, env = None, interval = RENEW_SECONDS ) : + + self.__snapshot = snapshot + self.__env = env + self.__interval = interval + self.__stop = threading.Event() + self.__lock = threading.Lock() + self.__results = [] + self.__count = 0 + self.__targets = {} + self.__offered = {} # offer_key: token, of the offers made + self.__thread = threading.Thread( target = self.__loop, name = "GafferOpenCodeAdvertiser", daemon = True ) + + def start( self ) : + + self.__thread.start() + return self + + @property + def results( self ) : + + with self.__lock : + return list( self.__results ) + + def advertiseOnce( self ) : + + port, token, scripts = self.__snapshot() + if token is None : + return None + offers = [ + { + "host" : "127.0.0.1", "port" : port, "token" : token, "script_handle" : handle, "label" : label, + "protocol" : PROTOCOL, "offer_key" : key, + } + for handle, label, key in scripts + ] + # A script closed since: its offer is withdrawn rather than left to expire. + keys = { offer["offer_key"] for offer in offers } + for key in [ key for key in self.__offered if key not in keys ] : + self.__withdraw( key, self.__offered.pop( key ) ) + results = self.__renew( offers ) if offers else [] + for offer in offers : + self.__offered[offer["offer_key"]] = token + with self.__lock : + self.__results = results + return results + + def __renew( self, offers ) : + + """Renews at the services that took the offers (one request per offer); discovers them again on a failure and + every DISCOVER_EVERY renewals (a discovery reads each service's info and the plugin's status first).""" + + from . import opencode_service # the kit's client, vendored into the payload + + self.__count += 1 + targets = self.__targets + if targets and self.__count % DISCOVER_EVERY : + results, failed = [], False + for name, ( service, directory ) in targets.items() : + for offer in offers : + try : + status, body = opencode_service.call( service, RPC_ID, "advertise", offer, directory, RENEW_TIMEOUT ) + outcome = opencode_service._outcome( "advertise", status, body ) + except ( OSError, ValueError ) : + outcome = "error" + results.append( { "service" : name, "label" : offer["label"], "outcome" : outcome } ) + failed = failed or outcome not in ( "accepted", "renewed" ) + if not failed : + return results + found = {} + results = [] + try : + services = opencode_service.discover( env = self.__env, timeout = RENEW_TIMEOUT ) + except ( OSError, ValueError ) : + services = [] + for service in services : + for directory in opencode_service.location_candidates( None, self.__env ) : + outcomes = [] + try : + deadline = time.monotonic() + RENEW_TIMEOUT + status, body = opencode_service._ready( service, RPC_ID, directory, deadline ) + if opencode_service._unavailable( status, body ) : + continue + if status != 200 : + outcomes = [ ( offer, "error" ) for offer in offers ] + else : + for offer in offers : + try : + status, body = opencode_service.call( + service, RPC_ID, "advertise", offer, directory, max( 0.5, deadline - time.monotonic() ) + ) + outcomes.append( ( offer, opencode_service._outcome( "advertise", status, body ) ) ) + except ( OSError, ValueError ) : + outcomes.append( ( offer, "error" ) ) + except ( OSError, ValueError ) : + outcomes = [ ( offer, "error" ) for offer in offers ] + for offer, outcome in outcomes : + results.append( { "service" : service.file, "label" : offer["label"], "outcome" : outcome } ) + if any( outcome in ( "accepted", "renewed" ) for _, outcome in outcomes ) : + found[service.file] = ( service, directory ) + break + else : + results.append( { "service" : service.file, "outcome" : "unavailable" } ) + self.__targets = found + return results + + def __withdraw( self, key, token ) : + + from . import opencode_service + + try : + opencode_service.withdraw_all( RPC_ID, key, token, env = self.__env ) + except ( OSError, ValueError ) : + pass + + def __loop( self ) : + + while not self.__stop.is_set() : + try : + results = self.advertiseOnce() or [] + except Exception : # the next renewal tries again + results = [] + taken = all( result.get( "outcome" ) in ( "accepted", "renewed" ) for result in results ) and results + # A renewal that failed is tried again soon: an offer lasts 15 s from the last one taken. + self.__stop.wait( self.__interval if taken else min( self.__interval, 1.0 ) ) + + def stop( self, token, withdraw = True ) : + + """Stops renewing; withdraws every offer from every service with the token it was announced with.""" + + from . import opencode_service + + self.__stop.set() + if self.__thread.is_alive() and self.__thread is not threading.current_thread() : + self.__thread.join( timeout = opencode_service.TIMEOUT * 2 ) + if not withdraw or token is None : + return [] + results = [] + for key in list( self.__offered ) : + try : + results += opencode_service.withdraw_all( RPC_ID, key, token, env = self.__env ) + except ( OSError, ValueError ) : + pass + self.__offered = {} + return results diff --git a/integrations/gaffer/host/python/GafferOpenCode/Gate.py b/integrations/gaffer/host/python/GafferOpenCode/Gate.py new file mode 100644 index 0000000..e2b22e5 --- /dev/null +++ b/integrations/gaffer/host/python/GafferOpenCode/Gate.py @@ -0,0 +1,129 @@ +"""Runs requests on Gaffer's main thread, and says which runs. + +In the interface, through GafferUI.EventLoop.executeOnUIThread (callable from any thread: it queues the call and +schedules it with a queued Qt invocation, without waiting for the main thread); in `gaffer python`, through pump(), +which the main thread calls. A request the main thread has not taken within QUEUE_WAIT_S is answered busy, naming +what runs there and since when. `running()` answers from any thread. Never one request inside another: a request +reached from a nested event loop (a dialog, processEvents) while another runs is put back and retried. +""" + +import queue +import threading +import time + +QUEUE_WAIT_S = 10.0 +RUN_LIMIT_S = 3600.0 +RETRY_S = 0.05 + + +class GateError( RuntimeError ) : + + def __init__( self, code, message, data = None ) : + + RuntimeError.__init__( self, message ) + self.code = code + self.data = data or {} + + +class UIThreadGate : + + def __init__( self, post = None ) : + + # post( callable ) schedules a call on the main thread (the interface's event loop); None: pump() serves. + self.__post = post + self.__mainThread = threading.get_ident() + self.__running = None + self.__queued = 0 + self.__count = threading.Lock() + self.__jobs = queue.Queue() + + def running( self ) : + + current = self.__running + if current is None : + return { "busy" : False, "queued" : self.__queued } + label, since = current + return { "busy" : True, "running" : label, "seconds" : round( time.monotonic() - since, 1 ), "queued" : self.__queued } + + def __invoke( self, label, callback ) : + + self.__running = ( label, time.monotonic() ) + try : + return callback() + finally : + self.__running = None + + def pump( self, seconds = 0.05, stop = None ) : + + """`gaffer python`: runs the queued requests on this (the main) thread for `seconds`, or until `stop()`.""" + + deadline = time.monotonic() + seconds + while time.monotonic() < deadline and not ( stop is not None and stop() ) : + try : + job = self.__jobs.get( timeout = min( 0.05, max( 0.0, deadline - time.monotonic() ) ) ) + except queue.Empty : + continue + job() + + def __schedule( self, job ) : + + if self.__post is None : + self.__jobs.put( job ) + else : + self.__post( job ) + + def run( self, callback, label = "a request" ) : + + if threading.get_ident() == self.__mainThread : + return self.__invoke( label, callback ) + started = threading.Event() + complete = threading.Event() + expired = threading.Event() + lock = threading.Lock() + result = [] + + def invoke() : + if self.__running is not None : + # Reached from a nested event loop while a request runs: later, never inside it. + threading.Timer( RETRY_S, lambda : self.__schedule( invoke ) ).start() + return + with lock : + if expired.is_set() : + return + started.set() + try : + result.append( ( True, self.__invoke( label, callback ) ) ) + except Exception as error : + result.append( ( False, error ) ) + finally : + complete.set() + + with self.__count : + self.__queued += 1 + try : + self.__schedule( invoke ) + if not started.wait( QUEUE_WAIT_S ) : + with lock : + if not started.is_set() : + expired.set() + now = self.running() + doing = ( + "it still runs {} ({} s so far): nothing was done for this request; ask again once it has " + "finished".format( now["running"], now["seconds"] ) + if now.get( "busy" ) + else "a modal dialog, a long computation or a script outside OpenCode holds it" + ) + raise GateError( + "ui_timeout", + "Gaffer's main thread did not take the request within {:.0f} s: {}".format( QUEUE_WAIT_S, doing ), + { "running" : now.get( "running" ), "seconds" : now.get( "seconds" ) }, + ) + finally : + with self.__count : + self.__queued -= 1 + if not complete.wait( RUN_LIMIT_S ) : + raise GateError( "ui_timeout", "Gaffer did not finish the request within {:.0f} s".format( RUN_LIMIT_S ) ) + succeeded, value = result[0] + if not succeeded : + raise value + return value diff --git a/integrations/gaffer/host/python/GafferOpenCode/Hints.py b/integrations/gaffer/host/python/GafferOpenCode/Hints.py new file mode 100644 index 0000000..4f3d4fa --- /dev/null +++ b/integrations/gaffer/host/python/GafferOpenCode/Hints.py @@ -0,0 +1,222 @@ +"""Hints for a failed execute_code run, computed inside Gaffer in the except block, before anything is undone. + +explain( error, code, filename, script ) returns a few lines saying what Gaffer really offers where the code went +wrong, or None. Each hint only reads: children of existing graph components, plug types and values, dir() of +classes and modules, a ProcessException's plug. Measured with Gaffer 1.7.1.0: + - a missing child: KeyError "'radiuss' is not a child of 'sphere'" + - a wrong value type: Boost.Python.ArgumentError "FloatPlug.setValue(FloatPlug, str) did not match ..." + - an unknown attribute: AttributeError (name, obj) + - a computation error: Gaffer.ProcessException, naming the plug ("node.plug : message") + - a refused connection: IECore.Exception 'Plug "ScriptNode.a.b" rejects input "ScriptNode.c.d".' +""" + +import difflib +import re + +MAX_CHARS = 1500 +MAX_ITEMS = 40 + +_CHILD = re.compile( r"^'([^']*)' is not a child of '([^']*)'$" ) +_ARGUMENTS = re.compile( r"Python argument types in\s+(\w+)\.(\w+)\(([^)]*)\)" ) +_REJECTS = re.compile( r'Plug "([^"]+)" rejects input "([^"]+)"' ) +_NAME = re.compile( r"name '(\w+)' is not defined" ) +GAFFER_MODULES = ( + "Gaffer", "GafferScene", "GafferImage", "GafferDispatch", "GafferOSL", "GafferCycles", "GafferUSD", "GafferArnold", + "GafferRenderMan", "GafferML", "GafferVDB", "IECore", "IECoreScene", "IECoreImage", "imath", +) + + +def closest( name, candidates, n = 5 ) : + + """The candidates closest to name: difflib matches (case-insensitive), then those containing it.""" + + pool = sorted( { str( c ) for c in candidates if c } ) + if not name or not pool : + return [] + lower = { c.lower() : c for c in pool } + found = [ lower[c] for c in difflib.get_close_matches( name.lower(), list( lower ), n = n, cutoff = 0.5 ) ] + needle = name.lower() + found += [ c for c in pool if c not in found and ( needle in c.lower() or ( len( c ) > 2 and c.lower() in needle ) ) ] + return found[:n] + + +def _bounded( names, limit = MAX_ITEMS ) : + + items = list( names ) + shown = ", ".join( items[:limit] ) + return shown + ( " ... ({} more)".format( len( items ) - limit ) if len( items ) > limit else "" ) + + +def _components( script ) : + + """Every node and plug of the script, depth first (bounded).""" + + import Gaffer + + found = [] + pending = [ script ] + while pending and len( found ) < 20000 : + component = pending.pop() + found.append( component ) + pending.extend( reversed( component.children( Gaffer.GraphComponent ) ) ) + return found + + +def _childHint( error, script ) : + + match = _CHILD.match( str( error.args[0] ) if error.args else "" ) + if not match or script is None : + return None + wanted, parentName = match.groups() + parents = [ c for c in _components( script ) if c.getName() == parentName and wanted not in c ] + if script.getName() == parentName or parentName == "ScriptNode" : + parents.insert( 0, script ) + for parent in parents[:3] : + names = parent.keys() + near = closest( wanted, names ) + lines = [ "{} ({}) has no child {!r}.".format( parent.relativeName( script ) or parent.getName(), parent.typeName(), wanted ) ] + if near : + lines.append( "Closest: " + ", ".join( near ) ) + lines.append( "Its children: " + _bounded( names ) ) + return "\n".join( lines ) + return None + + +def _valueType( className ) : + + import Gaffer + + cls = getattr( Gaffer, className, None ) + value = getattr( cls, "ValueType", None ) + if value is None : + return None + module = getattr( value, "__module__", "builtins" ) + return value.__name__ if module == "builtins" else "{}.{}".format( module, value.__name__ ) + + +def _argumentHint( error ) : + + text = str( error ) + match = _ARGUMENTS.search( text ) + if not match : + return None + cls, method, given = match.groups() + lines = [ "{}.{} was given ({}).".format( cls, method, given.strip() ) ] + if method == "setValue" : + value = _valueType( cls ) + if value : + lines.append( "{}.setValue takes a {} (the plug's ValueType).".format( cls, value ) ) + signature = text.split( "did not match C++ signature:", 1 ) + if len( signature ) == 2 : + lines.append( "C++ signature: " + " ".join( signature[1].split() )[:400] ) + return "\n".join( lines ) + + +def _attributeHint( error ) : + + name = getattr( error, "name", None ) + obj = getattr( error, "obj", None ) + if not name or obj is None : + return None + try : + names = [ n for n in dir( obj ) if not n.startswith( "_" ) ] + except Exception : + return None + near = closest( name, names ) + label = obj.__name__ if hasattr( obj, "__name__" ) and type( obj ).__name__ == "module" else type( obj ).__name__ + if not near : + return "{} has no {!r}.".format( label, name ) + return "{} has no {!r}. Closest: {}".format( label, name, ", ".join( near ) ) + + +def _processHint( error, script ) : + + plug = None + try : + plug = error.plug() + except Exception : + return None + if plug is None : + return None + node = plug.node() + lines = [] + try : + process = error.processType() + except Exception : + process = "compute" + if node is not None : + where = node.relativeName( script ) if script is not None and script.isAncestorOf( node ) else node.fullName() + lines.append( "Computing {} of {} ({}) failed ({}).".format( plug.getName(), where, node.typeName(), process ) ) + import Gaffer + + if isinstance( node, Gaffer.Expression ) : + try : + expression, language = node.getExpression() + lines.append( "Its {} expression: {}".format( language, expression[:400] ) ) + except Exception : + pass + for name in ( "fileName", "filePath" ) : + if name in node and hasattr( node[name], "getValue" ) : + try : + lines.append( "{}: {!r}".format( name, node[name].getValue() ) ) + except Exception : + pass + inputs = [] + for child in Gaffer.Plug.RecursiveInputRange( node ) : + source = child.getInput() + if source is not None and source.node() is not None and source.node() is not node : + inputs.append( "{} <- {}".format( child.relativeName( node ), source.node().getName() ) ) + if inputs : + lines.append( "Its inputs: " + _bounded( inputs, 12 ) ) + return "\n".join( lines ) or None + + +def _rejectsHint( error, script ) : + + match = _REJECTS.search( str( error ) ) + if not match or script is None : + return None + found = [] + for full in match.groups() : + path = full.split( ".", 1 )[1] if "." in full else full + plug = script.descendant( path ) + found.append( "{} is a {}".format( path, plug.typeName() if plug is not None else "missing plug" ) ) + return "; ".join( found ) + ": connect plugs of compatible types (an output of a node into an input of another)." + + +def _nameHint( error, namespace ) : + + match = _NAME.search( str( error ) ) + if not match : + return None + name = match.group( 1 ) + lines = [] + if name in GAFFER_MODULES : + lines.append( "{} is not imported here: import {}".format( name, name ) ) + near = closest( name, list( namespace or {} ) + list( GAFFER_MODULES ) ) + if near and not lines : + lines.append( "Closest names: " + ", ".join( near ) ) + return "\n".join( lines ) or None + + +def explain( error, code = None, filename = None, script = None, namespace = None ) : + + import Gaffer + + hint = None + try : + if isinstance( error, Gaffer.ProcessException ) : + hint = _processHint( error, script ) + elif isinstance( error, KeyError ) : + hint = _childHint( error, script ) + elif type( error ).__name__ == "ArgumentError" : + hint = _argumentHint( error ) + elif isinstance( error, AttributeError ) : + hint = _attributeHint( error ) + elif isinstance( error, NameError ) : + hint = _nameHint( error, namespace ) + elif "rejects input" in str( error ) : + hint = _rejectsHint( error, script ) + except Exception : + return None + return hint[:MAX_CHARS] if hint else None diff --git a/integrations/gaffer/host/python/GafferOpenCode/Host.py b/integrations/gaffer/host/python/GafferOpenCode/Host.py index c4f3b78..ce89659 100644 --- a/integrations/gaffer/host/python/GafferOpenCode/Host.py +++ b/integrations/gaffer/host/python/GafferOpenCode/Host.py @@ -22,7 +22,11 @@ import GafferScene import GafferUI +from . import Advertiser +from . import Gate +from . import Hints from . import Recipes +from . import Runner PROTOCOL_VERSION = 1 @@ -43,7 +47,7 @@ "render.cancel", "ui.view", ) -METHODS = ( "hello", "context", "sync", "apply", "cancel", "resume" ) +METHODS = ( "hello", "status", "context", "sync", "apply", "execute_code", "release", "cancel", "resume" ) class HostError( RuntimeError ) : @@ -82,7 +86,7 @@ def _jsonValue( value ) : "unknown_script" : "stale_handle", "revision_conflict" : "stale_revision", "unknown_method" : "method_not_found", "script_quarantined" : "quarantined", "rollback_failed" : "quarantined", "request_too_large" : "invalid_request", - "read_only" : "permission_denied", + "read_only" : "permission_denied", "ui_timeout" : "busy", } WIRE_MESSAGE_MAX = 2000 WIRE_DATA_MAX = 64 * 1024 @@ -337,6 +341,7 @@ def __init__( self, script, recipes = None ) : self.epoch = str( uuid.uuid4() ) self.revision = 1 self.quarantined = False + self.quarantine = None self.__cancellation = {} self.__trackingSuppressed = False self.__refsByObject = {} @@ -566,7 +571,7 @@ def apply( self, request, cancellationGeneration, requestWasCancelled ) : if requestWasCancelled or self.__cancellationState( sessionID )[1] or cancellationGeneration != self.__cancellationState( sessionID )[0] : raise HostError( "cancelled", "The current OpenCode turn was interrupted" ) if self.quarantined : - raise HostError( "script_quarantined", "Remote writes are disabled until the script is reattached" ) + raise HostError( "script_quarantined", Runner.QUARANTINED, { "quarantine" : self.quarantine or {} } ) commandID = str( request.get( "command_id", "" ) ) if not commandID : @@ -673,7 +678,13 @@ def actionSlot( unusedScript, unusedAction, stage ) : self.flush( cause = "rollback", commandID = commandID ) except Exception as rollbackError : self.quarantined = True - raise HostError( "rollback_failed", str( rollbackError ) ) + self.quarantine = { + "cause" : ( "failed apply: " + str( failure ) )[:200], + "release" : "read the script (gaffer_context), check what may have changed, then release the " + "quarantine with that read's revision", + "undo_error" : str( rollbackError )[:300], + } + raise HostError( "rollback_failed", str( rollbackError ), { "quarantine" : self.quarantine } ) self.__trackingSuppressed = False raise HostError( "execution_failed_rolled_back" if actionCount[0] else "execution_failed", @@ -742,6 +753,19 @@ def cancellationState( self, sessionID ) : return self.__cancellationState( sessionID ) + # For execute_code (Runner.py): changes seen since the last flush, the redo of an undone run, a new summary. + def hasPendingChanges( self ) : + + return bool( self.__dirtyNodes or self.__removedRefs ) + + def discardRedo( self ) : + + self.__discardRedo() + + def markSummary( self ) : + + self.__summaryDirty = True + def __discardRedo( self ) : barrier = Gaffer.Node( "__opencodeRollbackBarrier" ) @@ -1648,27 +1672,126 @@ def __focusChanged( self, unusedScript, node ) : self.__recent.append( self.__ref( node ) ) +def _tempDir() : + + return tempfile.gettempdir() + + +def documentName( script ) : + + try : + name = pathlib.Path( script["fileName"].getValue() ).name + except Exception : + name = "" + return ( name or script.getName() or "Untitled" )[:120] + + +def offerLabel( script ) : + + name = "".join( c for c in documentName( script ) if ord( c ) >= 32 and c != "\x7f" ).strip() or "Untitled" + return "Gaffer: " + name + + class Host : - def __init__( self, application ) : + """The installed part: the scripts of one Gaffer process (application root), each with its handle and offer. + + `ui`: requests run through Gaffer's event loop (the interface); otherwise through pump() (`gaffer python`).""" + + def __init__( self, application = None, root = None, ui = True ) : self.application = application - self.recipes = Recipes.Registry( application.root() ) + root = root if root is not None else application.root() + self.root = root + self.recipes = Recipes.Registry( root ) + self.ui = Gate.UIThreadGate( ( lambda job : GafferUI.EventLoop.executeOnUIThread( job ) ) if ui else None ) self.__statesByHandle = {} self.__handlesByScript = {} + self.__offers = {} # script handle: ( label, offer key ) + self.__offerCount = 0 self.__connections = [] - scripts = application.root()["scripts"] + capabilities = { + "methods" : list( METHODS ), + "operations" : [ { "name" : name, "undoable" : name not in ( "render.start", "render.cancel", "ui.view" ) } for name in OPERATIONS ], + "undo" : "transactional", + "background" : False, + } + # What hello says that does not change while Gaffer runs, read once here on the main thread. + self.__hello = { + "protocol" : PROTOCOL_VERSION, + "host_version" : HOST_VERSION, + "app_version" : Gaffer.About.versionString(), + "process" : { "pid" : os.getpid(), "temp_dir" : _tempDir() }, + "capabilities" : capabilities, + "capability_hash" : hashlib.sha256( json.dumps( capabilities, sort_keys = True, separators = ( ",", ":" ) ).encode( "utf-8" ) ).hexdigest(), + "operations" : list( OPERATIONS ), + "node_types" : list( self.recipes.nodeTypes() ), + "node_recipes" : self.recipes.catalogue(), + } + scripts = root["scripts"] for script in scripts.children( Gaffer.ScriptNode ) : self.__addScript( scripts, script ) self.__connections.append( scripts.childAddedSignal().connect( self.__addScript, scoped = True ) ) self.__connections.append( scripts.childRemovedSignal().connect( self.__removeScript, scoped = True ) ) + def close( self ) : + + self.__connections = [] + for state in list( self.__statesByHandle.values() ) : + state.close() + self.__statesByHandle = {} + self.__handlesByScript = {} + self.__offers = {} + def scriptHandle( self, script ) : return self.__handlesByScript.get( script ) + def offers( self ) : + + """( script_handle, label, offer_key ) of each open script, for the advertiser (any thread).""" + + return [ ( handle, label, key ) for handle, ( label, key ) in list( self.__offers.items() ) ] + + def offerOf( self, script ) : + + handle = self.__handlesByScript.get( script ) + return self.__offers.get( handle ) if handle is not None else None + + def stateOf( self, script ) : + + handle = self.__handlesByScript.get( script ) + return self.__statesByHandle.get( handle ) if handle is not None else None + + def quickHello( self, handle = None ) : + + """hello without Gaffer's main thread: OpenCode checks each new offer with it, one offer at a time, and a + hello waiting behind a long computation would hold every offer of the computer back. Without a + script_handle: this process and its first script.""" + + if handle : + state = self.__statesByHandle.get( str( handle ) ) + if state is None : + raise HostError( "unknown_script", "Unknown or expired script_handle" ) + else : + states = list( self.__statesByHandle.values() ) + state = states[0] if states else None + hello = dict( self.__hello ) + if state is not None : + label = self.__offers.get( state.handle, ( "", "" ) )[0] + hello.update( + script_handle = state.handle, script_epoch = state.epoch, revision = state.revision, + document = label[len( "Gaffer: " ):] if label.startswith( "Gaffer: " ) else label, + ) + return hello + def dispatch( self, method, params ) : + if method == "status" : + # Answered on the request's own thread, never through the main thread: what it runs now. + return self.ui.running() + if method == "hello" : + return self.quickHello( params.get( "script_handle" ) ) if method in ( "cancel", "resume" ) : handle = str( params.get( "script_handle", "" ) ) state = self.__statesByHandle.get( handle ) @@ -1695,26 +1818,26 @@ def execute() : state = self.__statesByHandle.get( handle ) if state is None : raise HostError( "unknown_script", "Unknown or expired script_handle" ) - if method == "hello" : - return state.hello() if method == "context" : return state.context( params ) if method == "sync" : return state.sync( params.get( "after_revision", 0 ) ) if method == "apply" : return state.apply( params, requestCancellation[0], requestCancellation[1] ) + if method == "execute_code" : + return Runner.executeCode( state, params, self.root, Hints.explain ) + if method == "release" : + return Runner.release( state, params ) raise HostError( "unknown_method", "Unknown Gaffer host method: " + method ) - def executeSafely() : - try : - return True, execute() - except Exception as error : - return False, error - - succeeded, result = GafferUI.EventLoop.executeOnUIThread( executeSafely, waitForResult = True ) - if not succeeded : - raise result - return result + if method not in METHODS : + raise HostError( "unknown_method", "Unknown Gaffer host method: " + method ) + if str( params.get( "script_handle", "" ) ) not in self.__statesByHandle : + raise HostError( "unknown_script", "Unknown or expired script_handle" ) + try : + return self.ui.run( execute, method ) + except Gate.GateError as error : + raise HostError( error.code, str( error ), error.data ) def __addScript( self, unusedParent, script ) : @@ -1723,11 +1846,35 @@ def __addScript( self, unusedParent, script ) : state = ScriptState( script, self.recipes ) self.__handlesByScript[script] = state.handle self.__statesByHandle[state.handle] = state + self.__offers[state.handle] = ( self.__uniqueLabel( script, state.handle ), Advertiser.offerKey( self.__offerCount ) ) + self.__offerCount += 1 + + # The offer's label follows the script's file (a save as, a load), read here on the main thread. + def fileNameSet( plug ) : + if plug.isSame( script["fileName"] ) and state.handle in self.__offers : + self.__offers[state.handle] = ( self.__uniqueLabel( script, state.handle ), self.__offers[state.handle][1] ) + + self.__connections.append( script.plugSetSignal().connect( fileNameSet, scoped = True ) ) + + def __uniqueLabel( self, script, handle ) : + + """The script's label, numbered when another script of this Gaffer has it (two untitled scripts): a panel + finds its script's offer by label.""" + + label = offerLabel( script ) + taken = { other for key, ( other, unusedKey ) in self.__offers.items() if key != handle } + number = 2 + unique = label + while unique in taken : + unique = "{} ({})".format( label, number ) + number += 1 + return unique def __removeScript( self, unusedParent, script ) : handle = self.__handlesByScript.pop( script, None ) state = self.__statesByHandle.pop( handle, None ) if handle is not None else None + self.__offers.pop( handle, None ) if state is not None : state.close() @@ -1754,6 +1901,12 @@ def info( self ) : def close( self ) : self.running = False + self.token = secrets.token_hex( 32 ) + # Wake a blocking accept before closing the listener on every platform. + try : + self.socket.shutdown( socket.SHUT_RDWR ) + except OSError : + pass try : self.socket.close() except OSError : @@ -1772,7 +1925,7 @@ def __handleClient( self, client ) : requestID = None try : - client.settimeout( 2 ) + client.settimeout( 130 ) buffer = b"" while b"\n" not in buffer : chunk = client.recv( 65536 ) diff --git a/integrations/gaffer/host/python/GafferOpenCode/Runner.py b/integrations/gaffer/host/python/GafferOpenCode/Runner.py new file mode 100644 index 0000000..0bc24ee --- /dev/null +++ b/integrations/gaffer/host/python/GafferOpenCode/Runner.py @@ -0,0 +1,199 @@ +"""execute_code and release: Python run in Gaffer on the attached script, grouped as one undo step, undone when it fails. + +A run that changes the script is one Gaffer.UndoScope. When it raises after a change, the step is undone and the +script compared with its serialisation before the run: restored, or the script is quarantined (changes refused, +reads and read_only runs allowed) until the agent read it and releases it with that read's revision. read_only runs +(the plugin's fixed programs, inspections) make no undo step: a change they make is reported. Hints (Hints.py) are +computed in the except block, before anything is undone. +""" + +import contextlib +import hashlib +import io +import json +import os +import traceback + +STDOUT_MAX = 64 * 1024 +TRACE_MAX = 16 * 1024 +FILENAME_MAX = 1024 +UNDO_STEP = "OpenCode: code" +QUARANTINED = ( + "A failed change could not be proven undone: changes are refused until you read the script (gaffer_context) " + "and release the quarantine with that read's revision (gaffer_context release_quarantine)." +) + + +class _Capture( io.StringIO ) : + + """stdout or stderr of one run, bounded.""" + + def write( self, text ) : + + if self.tell() < STDOUT_MAX * 2 : + return io.StringIO.write( self, text ) + return len( text ) + + +def fingerprint( script ) : + + """The script as Gaffer serialises it: every node, plug value and connection an undone change must give back.""" + + try : + return hashlib.sha256( script.serialise().encode( "utf-8" ) ).hexdigest() + except Exception as error : + return "unserialisable: {}".format( error ) + + +def _jsonRepr( value ) : + + return json.loads( json.dumps( value, default = repr ) ) + + +def _savedAt( script ) : + + try : + path = script["fileName"].getValue() + return os.path.getmtime( path ) if path and os.path.isfile( path ) else None + except Exception : + return None + + +def namespace( script, application ) : + + import imath + import IECore + import Gaffer + import GafferDispatch + import GafferImage + import GafferScene + + return { + "Gaffer" : Gaffer, "GafferScene" : GafferScene, "GafferImage" : GafferImage, "GafferDispatch" : GafferDispatch, + "IECore" : IECore, "imath" : imath, "script" : script, "application" : application, + "__name__" : "__opencode__", "result" : {}, + } + + +def executeCode( state, params, application = None, explain = None ) : + + import Gaffer + from .Host import HostError + + code = params.get( "code" ) + if not isinstance( code, str ) or not code.strip() : + raise HostError( "invalid_request", "code is required" ) + readOnly = params.get( "read_only" ) is True + if state.quarantined and not readOnly : + raise HostError( "script_quarantined", QUARANTINED, { "quarantine" : state.quarantine or {} } ) + filename = params.get( "filename" ) + if filename is not None and ( + not isinstance( filename, str ) or not 0 < len( filename ) <= FILENAME_MAX or any( ord( c ) < 32 for c in filename ) + ) : + raise HostError( "invalid_request", "filename must be a file path of at most 1024 characters" ) + + script = state.script + scope = namespace( script, application ) + if filename : + scope["__file__"] = filename + state.flush() + revisionBefore = state.revision + written = _savedAt( script ) + before = None if readOnly else fingerprint( script ) + actions = [ 0 ] + + def counted( unusedScript, unusedAction, stage ) : + if stage == Gaffer.Action.Stage.Do : + actions[0] += 1 + + connection = script.actionSignal().connect( counted, scoped = True ) + out, err = _Capture(), _Capture() + error = trace = hint = None + group = contextlib.nullcontext() if readOnly else Gaffer.UndoScope( script, mergeGroup = "" ) + try : + with group : + try : + with contextlib.redirect_stdout( out ), contextlib.redirect_stderr( err ) : + exec( compile( code, filename or "", "exec" ), scope ) + except BaseException as exc : # SystemExit from supplied code must not quit Gaffer + error = "{}: {}".format( type( exc ).__name__, exc )[:2000] + trace = traceback.format_exc()[-TRACE_MAX:] + if explain is not None : + try : + hint = explain( exc, code, filename, script, scope ) + except Exception : # a hint never hides the error + hint = None + finally : + del connection + + saved = ( lambda now : now is not None and now != written )( _savedAt( script ) ) + changed = actions[0] > 0 or state.hasPendingChanges() or saved + restored = None + if error is not None and actions[0] and not readOnly : + rollback = None + try : + if not script.undoAvailable() : + raise RuntimeError( "the run left no undo item to undo" ) + script.undo() + state.discardRedo() + except Exception as undoError : + rollback = undoError + restored = rollback is None and before == fingerprint( script ) + if not restored : + state.quarantined = True + state.quarantine = { + "cause" : ( "failed code: " + error )[:200], + "release" : "read the script (gaffer_context), check what may have changed, then release the " + "quarantine with that read's revision", + **( { "undo_error" : str( rollback )[:300] } if rollback is not None else {} ), + } + state.markSummary() + state.flush( cause = "rollback" ) + else : + state.flush( cause = "opencode" ) + + output = { + "status" : "failed" if error else "executed", + "script_handle" : state.handle, + "result" : _jsonRepr( scope.get( "result", {} ) ), + "stdout" : out.getvalue()[:STDOUT_MAX], + "stderr" : err.getvalue()[:STDOUT_MAX], + "truncated" : len( out.getvalue() ) > STDOUT_MAX or len( err.getvalue() ) > STDOUT_MAX, + "revision_before" : revisionBefore, + "revision_after" : state.revision, + "changed" : changed, + } + if changed and not readOnly and error is None and actions[0] : + output["undo_step"] = UNDO_STEP + if saved : + output["saved"] = True + if readOnly and actions[0] : + output["warnings"] = [ "OpenCode: this read_only run changed the script; no undo step holds it." ] + if error : + output.update( message = error, traceback = trace ) + if hint : + output["hint"] = hint + if restored is not None : + output["restored"] = restored + if not restored : + output["quarantine"] = state.quarantine + return output + + +def release( state, params ) : + + from .Host import HostError + + if not state.quarantined : + return { "released" : False, "revision" : state.revision } + revision = params.get( "revision" ) + state.flush() + if not isinstance( revision, int ) or isinstance( revision, bool ) or revision != state.revision : + raise HostError( + "revision_conflict", + "Read the script again (gaffer_context) and release with its revision.", + { "current_revision" : state.revision }, + ) + state.quarantined = False + state.quarantine = None + return { "released" : True, "revision" : state.revision } diff --git a/integrations/gaffer/host/python/GafferOpenCode/__init__.py b/integrations/gaffer/host/python/GafferOpenCode/__init__.py index 9155837..631d140 100644 --- a/integrations/gaffer/host/python/GafferOpenCode/__init__.py +++ b/integrations/gaffer/host/python/GafferOpenCode/__init__.py @@ -1,30 +1,123 @@ -from .Host import Host, HostServer +"""OpenCode Link for Gaffer: the installed part. + +Started with Gaffer's interface by startup/gui/openCode.py (initialise): it listens on 127.0.0.1 and offers each +open script to the OpenCode services every 5 seconds, so any conversation (CodeNomad, the terminal, the OpenCode +panel) can attach it. OPENCODE_GAFFER_OFFER=0 in Gaffer's environment keeps it from offering them. A background +`gaffer python` (verification, renders: OPENCODE_GAFFER_ROLE) never starts it. +""" + +import atexit +import os + +from .Host import Host, HostError, HostServer, ScriptState __host = None __server = None +__advertiser = None +__panelOwners = set() + + +def _snapshot() : + host, server = __host, __server + if host is None or server is None or not server.running : + return None, None, [] + return server.port, server.token, host.offers() -def initialise( application ) : - global __host, __server +def initialise( application = None, advertise = None, root = None, ui = True ) : + + """Starts the host once per Gaffer process; with advertise (by default unless OPENCODE_GAFFER_OFFER=0 or a + background role), offers its scripts to OpenCode.""" + + global __host, __server, __advertiser if __host is not None : return - host = Host( application ) + host = Host( application, root = root, ui = ui ) server = HostServer( host ) __host = host __server = server + if advertise is None : + advertise = os.environ.get( "OPENCODE_GAFFER_OFFER", "1" ) != "0" and not os.environ.get( "OPENCODE_GAFFER_ROLE" ) + if advertise : + from .Advertiser import Advertiser + + __advertiser = Advertiser( _snapshot ).start() + + +def pump( seconds = 0.05, stop = None ) : + + """`gaffer python`: serve the queued requests on the main thread (the interface serves them by itself).""" + + if __host is not None : + __host.ui.pump( seconds, stop ) + + +def shutdown() : + + global __host, __server, __advertiser + advertiser, server = __advertiser, __server + token = server.token if server is not None else None + if advertiser is not None : + advertiser.stop( token ) + if server is not None : + server.close() + if __host is not None : + __host.close() + __advertiser = None + __server = None + __host = None + __panelOwners.clear() + + +atexit.register( shutdown ) + + +def acquirePanel( channel ) : + + """A panel uses the host (started with Gaffer's interface).""" + + __panelOwners.add( channel ) + + +def releasePanel( channel ) : + + """A panel closed: the host and its offers stay, for the other conversations attached to this Gaffer.""" + + __panelOwners.discard( channel ) def hostInfo() : if __server is None : - raise RuntimeError( "Gaffer OpenCode host has not been initialised" ) + raise RuntimeError( "OpenCode Link for Gaffer has not been initialised" ) return __server.info() +def offerInfo( script ) : + + """What a panel needs to find this script's offer among the offered hosts.""" + + if __host is None : + raise RuntimeError( "OpenCode Link for Gaffer has not been initialised" ) + label, key = __host.offerOf( script ) or ( None, None ) + return { "label" : label, "offer_key" : key, "advertising" : __advertiser is not None } + + def scriptHandle( script ) : if __host is None : - raise RuntimeError( "Gaffer OpenCode host has not been initialised" ) + raise RuntimeError( "OpenCode Link for Gaffer has not been initialised" ) return __host.scriptHandle( script ) + + +def currentHost() : + + return __host + + +__all__ = [ + "Host", "HostError", "HostServer", "ScriptState", "acquirePanel", "currentHost", "hostInfo", "initialise", + "offerInfo", "pump", "releasePanel", "scriptHandle", "shutdown", +] diff --git a/integrations/gaffer/host/python/GafferOpenCodeUI/Client.py b/integrations/gaffer/host/python/GafferOpenCodeUI/Client.py index 3168536..2e1cb38 100644 --- a/integrations/gaffer/host/python/GafferOpenCodeUI/Client.py +++ b/integrations/gaffer/host/python/GafferOpenCodeUI/Client.py @@ -29,13 +29,13 @@ def __init__( self ) : self.__process.finished.connect( self.__finished ) self.__process.errorOccurred.connect( self.__processError ) self.__channels = {} - self.__connectParams = {} self.__pending = [] self.__stdout = b"" self.__nextRequest = 0 self.__start() - def register( self, callback, connectParams ) : + ## One panel's events; the callback is held weakly. + def register( self, callback ) : channel = str( uuid.uuid4() ) try : @@ -43,8 +43,6 @@ def register( self, callback, connectParams ) : except TypeError : callbackRef = weakref.ref( callback ) self.__channels[channel] = callbackRef - self.__connectParams[channel] = connectParams - self.send( channel, "connect", connectParams ) return channel def close( self, channel ) : @@ -53,7 +51,6 @@ def close( self, channel ) : return self.send( channel, "close", {} ) self.__channels.pop( channel, None ) - self.__connectParams.pop( channel, None ) def send( self, channel, method, params ) : @@ -81,22 +78,18 @@ def __start( self ) : if self.__process.state() != QtCore.QProcess.NotRunning : return - bun = os.environ.get( "NEURALNOMADS_GAFFER_BUN" ) or shutil.which( "bun" ) - if not bun : + node = os.environ.get( "NEURALNOMADS_GAFFER_NODE" ) or shutil.which( "node" ) + if not node : self.__pending = [] - self.__emitAll( "error", { "message" : "Install Bun or set NEURALNOMADS_GAFFER_BUN" } ) + self.__emitAll( "error", { "message" : "Install Node.js 22 or set NEURALNOMADS_GAFFER_NODE" } ) return - sidecar = pathlib.Path( - os.environ.get( - "NEURALNOMADS_GAFFER_SIDECAR", - str( pathlib.Path( __file__ ).resolve().parents[2] / "sidecar.js" ), - ) - ) + # Built into the installed part, beside the Python packages. + sidecar = pathlib.Path( __file__ ).resolve().parents[2] / "sidecar.cjs" if not sidecar.is_file() : self.__pending = [] - self.__emitAll( "error", { "message" : "OpenCode Gaffer sidecar not found: " + str( sidecar ) } ) + self.__emitAll( "error", { "message" : "OpenCode Link sidecar is missing; reinstall the application part." } ) return - self.__process.setProgram( bun ) + self.__process.setProgram( node ) self.__process.setArguments( [ str( sidecar ) ] ) self.__process.setWorkingDirectory( str( sidecar.parent.parent ) ) self.__process.start() @@ -149,7 +142,6 @@ def __emit( self, channel, name, data ) : callback = callbackRef() if callbackRef is not None else None if callback is None : self.__channels.pop( channel, None ) - self.__connectParams.pop( channel, None ) return callback( name, data ) diff --git a/integrations/gaffer/host/python/GafferOpenCodeUI/OpenCodeEditor.py b/integrations/gaffer/host/python/GafferOpenCodeUI/OpenCodeEditor.py index 6ad2c1a..339c6ec 100644 --- a/integrations/gaffer/host/python/GafferOpenCodeUI/OpenCodeEditor.py +++ b/integrations/gaffer/host/python/GafferOpenCodeUI/OpenCodeEditor.py @@ -2,7 +2,7 @@ import hashlib import os import pathlib -import uuid +import weakref import IECore import imath @@ -108,16 +108,15 @@ def __init__( self, scriptNode, **kw ) : self.__permissionRequest = None self.__streaming = False self.__busy = False + # A plain client: Gaffer offers this script to OpenCode by itself (GafferOpenCode), whether or not a + # panel is open; the panel only attaches its own conversation to that offer. self.__client = None self.__channel = None - hostInfo = GafferOpenCode.hostInfo() - scriptHandle = GafferOpenCode.scriptHandle( scriptNode ) - if not scriptHandle : + self.__active = False + self.__connected = False + if not GafferOpenCode.scriptHandle( scriptNode ) : raise RuntimeError( "OpenCodeEditor requires a ScriptNode registered with the Gaffer application" ) - self.__connectParams = dict( hostInfo ) - self.__connectParams["script_handle"] = scriptHandle - self.__unsavedKey = uuid.uuid4().hex - self.__updateConnectParams() + self.__scriptKey = self.__currentScriptKey() self.__scriptConnections = [ scriptNode.plugSetSignal().connect( Gaffer.WeakMethod( self.__scriptPlugSet ), scoped = True ), ] @@ -126,32 +125,65 @@ def __repr__( self ) : return "GafferOpenCodeUI.OpenCodeEditor( scriptNode )" + ## Closing the editor releases its conversation only: the script's offer stays for every other conversation. + @staticmethod + def __release( client, channel ) : + + try : + client.close( channel ) + finally : + GafferOpenCode.releasePanel( channel ) + + def __ensureChannel( self ) : + + if self.__channel is None : + self.__client = ClientProcess.acquire() + self.__channel = self.__client.register( self.__event ) + weakref.finalize( self, OpenCodeEditor.__release, self.__client, self.__channel ) + return self.__channel + def __toggleConnection( self, unusedButton = None ) : - if self.__channel is not None : - self.__client.close( self.__channel ) - self.__channel = None - self.__connectButton.setText( "Connect" ) - self.__statusLabel.setText( "DISCONNECTED" ) - self.__sendButton.setEnabled( False ) - self.__newButton.setEnabled( False ) + if self.__active : + self.__disconnect() + else : + self.__connect( "connect" ) + + def __connect( self, method ) : + + params = self.__connectParams() + if params is None : return - self.__client = ClientProcess.acquire() - self.__updateConnectParams() - self.__channel = self.__client.register( self.__event, self.__connectParams ) + channel = self.__ensureChannel() + GafferOpenCode.acquirePanel( channel ) + self.__active = True + self.__connected = False self.__connectButton.setText( "Disconnect" ) self.__newButton.setEnabled( True ) + self.__statusLabel.setText( "CONNECTING" ) + self.__sendButton.setEnabled( False ) + self.__client.send( channel, method, params ) - def __del__( self ) : + def __disconnect( self ) : - try : - self.__client.close( self.__channel ) - except Exception : - pass + self.__active = False + self.__connected = False + if self.__channel is not None : + self.__client.send( self.__channel, "close", {} ) + GafferOpenCode.releasePanel( self.__channel ) + self.__connectButton.setText( "Connect" ) + self.__statusLabel.setText( "DISCONNECTED" ) + self.__sendButton.setEnabled( False ) + self.__stopButton.setEnabled( False ) + self.__newButton.setEnabled( False ) + self.__modelButton.setEnabled( False ) def __event( self, name, data ) : + if not self.__active and name != "error" : + return # a reply to a connection this editor has since closed if name == "session" : + self.__connected = True self.__sessionLabel.setText( "SESSION " + str( data.get( "id", "" ) )[-8:] ) self.__sessionLabel.setToolTip( "Workspace: " + str( data.get( "workspace", "" ) ) ) self.__setModel( str( data.get( "model", "" ) ) ) @@ -185,20 +217,18 @@ def __event( self, name, data ) : self.__statusLabel.setText( state.upper() ) self.__busy = state in ( "busy", "retry" ) self.__stopButton.setEnabled( self.__busy ) - self.__sendButton.setEnabled( not self.__busy ) + self.__sendButton.setEnabled( self.__connected and not self.__busy ) self.__prompt.setEnabled( not self.__busy ) - self.__modelButton.setEnabled( bool( self.__models ) and not self.__busy ) - elif name == "graph_revision" : - self.__statusLabel.setToolTip( "Gaffer graph revision " + str( data.get( "revision" ) ) ) + self.__modelButton.setEnabled( self.__connected and bool( self.__models ) and not self.__busy ) elif name == "error" : self.__append( "error", data.get( "message", "Unknown OpenCode error" ) ) self.__statusLabel.setText( "ERROR" ) self.__busy = False self.__streaming = False self.__stopButton.setEnabled( False ) - self.__sendButton.setEnabled( True ) + self.__sendButton.setEnabled( self.__connected ) self.__prompt.setEnabled( True ) - self.__modelButton.setEnabled( bool( self.__models ) ) + self.__modelButton.setEnabled( self.__connected and bool( self.__models ) ) def __setModel( self, model ) : @@ -227,37 +257,55 @@ def __modelMenuDefinition( self ) : def __selectModel( self, unusedMenu = None, model = "" ) : - if self.__busy or not model or model == self.__model : + if self.__busy or not self.__connected or not model or model == self.__model : return self.__client.send( self.__channel, "switch_model", { "model" : model } ) - def __updateConnectParams( self ) : + ## The conversation marked for this script is taken back at Connect: a saved script by its file, an unsaved + # one while it stays open. + def __currentScriptKey( self ) : + + scriptNode = self.scriptNode() + fileName = str( scriptNode["fileName"].getValue() ) + if not fileName : + return "untitled:" + GafferOpenCode.scriptHandle( scriptNode ) + return "document:" + hashlib.sha256( str( pathlib.Path( fileName ).resolve() ).encode( "utf-8" ) ).hexdigest() + + def __connectParams( self ) : scriptNode = self.scriptNode() + offer = GafferOpenCode.offerInfo( scriptNode ) + if not offer.get( "advertising" ) or not offer.get( "label" ) : + self.__append( "error", "OpenCode Link does not offer this script to OpenCode." ) + return None fileName = str( scriptNode["fileName"].getValue() ) - directory = str( pathlib.Path( fileName ).resolve().parent ) if fileName else os.getcwd() - self.__connectParams.update( { - "script_key" : "document:" + hashlib.sha256( str( pathlib.Path( fileName ).resolve() ).encode( "utf-8" ) ).hexdigest() if fileName else "untitled:" + self.__unsavedKey, + params = dict( GafferOpenCode.hostInfo() ) + params.update( { + "script_handle" : GafferOpenCode.scriptHandle( scriptNode ), + "offer_label" : offer["label"], + "script_key" : self.__scriptKey, "script_name" : pathlib.Path( fileName ).name if fileName else scriptNode.getName(), - "directory" : directory, + "directory" : str( pathlib.Path( fileName ).resolve().parent ) if fileName else os.getcwd(), } ) + return params def __scriptPlugSet( self, plug ) : if not plug.isSame( self.scriptNode()["fileName"] ) : return - previousKey = self.__connectParams["script_key"] - self.__updateConnectParams() - if self.__connectParams["script_key"] != previousKey and self.__channel is not None : - self.__toggleConnection() + previousKey = self.__scriptKey + self.__scriptKey = self.__currentScriptKey() + # Another file is another script's conversation: Connect takes that one. + if self.__scriptKey != previousKey and self.__active : + self.__disconnect() def __newSession( self, unusedButton ) : - self.__client.send( self.__channel, "new_session", self.__connectParams ) + self.__connect( "new_session" ) def __send( self, unusedWidget ) : - if self.__busy or self.__channel is None : + if self.__busy or not self.__connected : return text = self.__prompt.getText().strip() if not text : diff --git a/integrations/gaffer/host/startup/gui/openCode.py b/integrations/gaffer/host/startup/gui/openCode.py index 94ca0fb..121db0c 100644 --- a/integrations/gaffer/host/startup/gui/openCode.py +++ b/integrations/gaffer/host/startup/gui/openCode.py @@ -1,15 +1,22 @@ +# OpenCode Link for Gaffer: starts the installed part with Gaffer's interface, so each open script is offered to +# OpenCode without any panel, and registers the OpenCode panel. Never fails Gaffer's startup. import pathlib import sys -import GafferUI +extensionRoot = pathlib.Path( __file__ ).resolve().parents[2] +pythonRoot = str( extensionRoot / "python" ) +if pythonRoot not in sys.path : + sys.path.insert( 0, pythonRoot ) -extensionRoot = pathlib.Path(__file__).resolve().parents[2] -pythonRoot = str(extensionRoot / "python") -if pythonRoot not in sys.path: - sys.path.insert(0, pythonRoot) +try : + import GafferOpenCode + GafferOpenCode.initialise( application ) +except Exception as error : + print( "OpenCode Link did not start: {}".format( error ) ) -import GafferOpenCode -GafferOpenCode.initialise(application) - -import GafferOpenCodeUI -GafferUI.Layouts.acquire(application.root()).registerEditor("OpenCode for Gaffer") +try : + import GafferUI + import GafferOpenCodeUI + GafferUI.Layouts.acquire( application.root() ).registerEditor( "OpenCode for Gaffer" ) +except Exception as error : + print( "The OpenCode panel is unavailable: {}".format( error ) ) diff --git a/integrations/gaffer/plugin/README.md b/integrations/gaffer/plugin/README.md index de4cdd4..72a645c 100644 --- a/integrations/gaffer/plugin/README.md +++ b/integrations/gaffer/plugin/README.md @@ -1,39 +1,60 @@ -# OpenCode for Gaffer +# OpenCode for Gaffer (preview) -Inspect and edit an open Gaffer graph through an authenticated loopback host. Works in every OpenCode client, including the terminal UI and CodeNomad. The optional Gaffer panel remains an OpenCode client. Preview, draft 0.2; Gaffer >=1.6. +Read, change and check the scripts open in Gaffer from any OpenCode conversation: CodeNomad, the terminal, the desktop app, or the **OpenCode for Gaffer** panel inside Gaffer. Gaffer 1.6 or later and OpenCode 2.2 or later are required; the panel also needs Node.js 22 or later. ## Install -Add the plugin to the **Global** OpenCode `plugins` list: +Add the plugin to the **global** OpenCode configuration (in CodeNomad: from the reviewed catalogue, in Plugins): ```jsonc { - "plugins": ["github:NeuralNomadsAI/opencode-integrations#main::path:integrations/gaffer/plugin"] + "$schema": "https://opencode.ai/config.json", + "plugins": [{ "package": "@neuralnomads/opencode-gaffer@^0.3" }] } ``` -The npm package name is `@neuralnomads/opencode-gaffer`; npm publication is a separate maintainer-approved action. Build a Git checkout with `npm ci && npm run build` before loading its local plugin directory. +Then install OpenCode Link, the part that runs inside Gaffer: `gaffer_install_host` (it asks permission), or CodeNomad's install action. It writes the loader `neuralnomadsOpenCode.py` into Gaffer's user startup folder (`~/gaffer/startup/gui`; `hostStartupDir` in the plugin's options names another one), which starts a checksummed copy of OpenCode Link under `%LOCALAPPDATA%/NeuralNomads/opencode-gaffer/hosts/` (`$XDG_STATE_HOME/neuralnomads/opencode-gaffer/hosts/` elsewhere). Installing again is idempotent; an existing loader not written by this integration is left untouched and the installation refused. To remove it, delete that loader. -Install the embedded application part explicitly with the permission-gated `gaffer_install_host` tool (or RPC `installHost` after client confirmation). It verifies the bundled ZIP's SHA-256, copies it into the integration's content-addressed state directory, and adds its managed loader to `~/gaffer/startup/gui/neuralnomadsOpenCode.py`. Set plugin option `hostStartupDir` (environment fallback `NEURALNOMADS_GAFFER_HOST_STARTUP_DIR`) for a different native Gaffer GUI startup directory. Existing unmanaged files are refused. Restart Gaffer to load the application part. To remove it, delete only that managed loader; content-addressed payloads can be retained offline. +Restart Gaffer: OpenCode Link starts with Gaffer's interface and offers each open script to OpenCode every 5 seconds, labelled with its file (`Gaffer: shot.gfr`; two scripts with the same file are numbered). `OPENCODE_GAFFER_OFFER=0` in Gaffer's environment keeps it from offering them. A background `gaffer python` (verification, renders) never offers itself. -The state directory is `$XDG_STATE_HOME/neuralnomads/opencode-gaffer` (default `~/.local/state/neuralnomads/opencode-gaffer`), or `%LOCALAPPDATA%/NeuralNomads/opencode-gaffer` on Windows. No file references the npm cache. The host and plugin share the package version and MIT licence. Gaffer's BSD-licensed APIs are used without redistributing its source. +An earlier OpenCode installation (`~/gaffer/startup/gui/openCode.py` and the `~/gaffer/OpenCode` folder) registers another editor and loads a package of the same name next to OpenCode Link: `gaffer_hosts` and CodeNomad's status say so. Remove them, then restart Gaffer. -## Connect +## Use -Start OpenCode yourself. Open the **OpenCode for Gaffer** editor in Gaffer and press **Connect** to offer its script. **Disconnect** or closing the panel withdraws the offer. The authenticated host listens only on `127.0.0.1` at an ephemeral port; its 256-bit process token stays in memory. Offers expire after 15 seconds and renew every 5 seconds. No RPC output or event contains credentials, the port or the script handle. An application closed without inspection reports host state `unknown`. +The first Gaffer tool call of a conversation attaches the only script offered when no other conversation uses it; otherwise `gaffer_hosts` lists the scripts offered and `gaffer_hosts attach` attaches one (permission `gaffer_attach`). An attachment is kept across an OpenCode restart. A working method joins the conversation when it first uses a Gaffer script. -The panel requires **Bun >=1.3 on PATH** (or `NEURALNOMADS_GAFFER_BUN` pointing to its executable). It runs the **compiled, self-contained `sidecar.js` shipped inside the host payload**, beside the Python host, rather than TypeScript in the npm cache. `NEURALNOMADS_GAFFER_SIDECAR` can explicitly override that path. The sidecar discovers a running service and never launches a daemon. The matching `@opencode/client` 2.2.5 is bundled into it. If your daemon uses a custom `OPENCODE_CONFIG_DIR`, set `NEURALNOMADS_GAFFER_GLOBAL_CONFIG_DIR` in Gaffer's environment to that daemon's Global configuration directory. +| Tool | | +| --------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `gaffer_hosts` | The scripts offered, which conversation holds each; `attach`, `detach`. | +| `gaffer_context` | A bounded read: revision, file, selection, focus, node cards; `release_quarantine` after a failed change. | +| `gaffer_execute_code` | Python in Gaffer on the attached script (`script`, `Gaffer`, `GafferScene`, ...), as code or a script file; permission `gaffer_execute_code` on every run. | +| `gaffer_inspect` | Search node classes, plugs and descriptions; a class with its plugs; a live node (plugs, values, errors, scene); a Python symbol. | +| `gaffer_verify` | Measured criteria, computation errors; `saved: true` reads the saved file in a background `gaffer python`. | +| `gaffer_save` | Save the script, or a copy that leaves the script's file name. | +| `gaffer_render` | A task node (a render, an image writer) executed by a background `gaffer python`, followed as a task (`task`, `wait_s`, `cancel`). | +| `gaffer_apply` | Typed graph edits (`node.create`, `node.rename`, `plug.set_value`, `graph.connect`, `graph.delete`, ...) as one undo step, permission `gaffer_write`. | +| `gaffer_install_host` | Install or update OpenCode Link. | -From another client, use `gaffer_hosts` and `gaffer_attach` with the offered opaque ID. Both require the named `gaffer_attach` permission. `gaffer_context` returns bounded reads and the current revision; `gaffer_apply` uses that exact revision, a unique command ID, 1–50 operations, native preflight, verification and transactional undo. Render and Viewer actions are separate side-effecting intents. There is no arbitrary-code tool. Detached conversations require an explicit reconnect; polling only renews offers, never silently reattaches them. +A code run that changes the script is one undo step (`OpenCode: code`); when it fails, the step is undone and the script compared with what it was. When that cannot be proven, changes are refused (reads and checks still run) until the agent reads the script and releases the quarantine with that read's revision. A failure says what Gaffer knows: the closest plug or child names, the closest classes of a module, the value type a plug's `setValue` takes, the plug a computation failed on with the node's expression or file, the types of two plugs that cannot connect. -Service discovery validates `service.json` under `$XDG_STATE_HOME/opencode` (default `~/.local/state/opencode`) before authentication: literal HTTP `127.0.0.1` or `::1`, with matching PID and version from `/api/info`. Probes and subsequent requests bypass proxies and refuse redirects. Each Render or Viewer intent contains one operation and no postconditions; its result declares `side_effect: true` and `undoable: false`. Render previews from individual jobs and job lists are returned as image attachments. +A call that outlasts its wait says whether Gaffer still runs it (OpenCode Link answers `status` off Gaffer's main thread). Gaffer writes no crash report: a Gaffer that closed or crashed is said as closed, one alive without renewing its offers as no longer offering them, and no other Gaffer attaches itself in its place. -## In CodeNomad +## The OpenCode for Gaffer panel -Install the Global plugin in Plugins, install its application part after confirmation, then select the open script in **Status › Gaffer** and attach the displayed conversation. Detach affects that conversation only. The in-app panel can also keep its own conversation attached. +The **OpenCode for Gaffer** editor is a conversation inside Gaffer. **Connect** attaches the panel's conversation to its script's own offer, resuming the conversation it had for this script; **New** starts another. Closing the panel releases its conversation only: the script stays offered and other conversations attached to it keep it. The panel and other conversations (CodeNomad's, for instance) can be attached to the same script together; both write to it, each change checked against its revision. The panel's conversation lives in the script's folder: the first Connect there waits while OpenCode opens that folder and loads its plugins. A failed Connect says which step failed and why. Node.js is found on `PATH`; `NEURALNOMADS_GAFFER_NODE` names another one. -## Build and checks +## Limits -`npm run build:host --workspace @neuralnomads/opencode-gaffer` deterministically compiles the sidecar, archives the Python host and panel, and refreshes the marker digest. `npm run build` verifies the archive before compiling the plugin. The generic protected publication workflow resolves `gaffer`, attests its archive and supports trusted publishing; it must be approved separately. +- Gaffer records undo only on its main thread: OpenCode Link runs every request there through Gaffer's event loop (in `gaffer python`, a script serves them with `GafferOpenCode.pump()`). A request the main thread has not taken within 10 s, because another one still runs there, is answered with what runs and since when; nothing is done for it. +- `gaffer_verify` with `saved` only starts its background `gaffer python` from Gaffer: the file is read while Gaffer stays free, `timeout_s` bounds the read and a stopped read says what it was doing. +- Node and plug descriptions come from Gaffer's UI modules, which `gaffer_inspect` imports (no window opens). -Run `node --test integrations/gaffer/test/*.test.ts`. Set `GAFFER_TEST_BIN` to Gaffer's launcher to exercise the real installed application in a private HOME/TEMP; the normal suite tests the actual authenticated host listener under a stub application as well as the plugin's RPC handlers. See [test/MANUAL.md](../test/MANUAL.md). +## Safety + +OpenCode Link listens on `127.0.0.1` only, on an ephemeral port, and authenticates every request with a process-random 256-bit token that lives in memory and private pipes only. Credentials, ports and script handles stay out of status outputs and events. Loading the plugin opens no socket and starts no process. + +## Tests + +`node --test integrations/gaffer/test/*.test.ts` runs the plugin, the panel's sidecar and the listener under a stub application. With `GAFFER_TEST_BIN` naming Gaffer's launcher (`gaffer.cmd` on Windows), `test/python/real_host.py` runs OpenCode Link in a real Gaffer (its event loop offscreen: code runs, undo, hints, quarantine, the busy answer, the offers) and `test/programs_gaffer.py` the plugin's programs, each with private HOME, USERPROFILE and temp folders. + +Publication uses the repository's manually approved OIDC workflow and `gaffer-v` tags. diff --git a/integrations/gaffer/plugin/host/gaffer/opencode_link-0.2.0.zip b/integrations/gaffer/plugin/host/gaffer/opencode_link-0.2.0.zip deleted file mode 100644 index fad9e8a..0000000 Binary files a/integrations/gaffer/plugin/host/gaffer/opencode_link-0.2.0.zip and /dev/null differ diff --git a/integrations/gaffer/plugin/host/gaffer/opencode_link-0.3.0.zip b/integrations/gaffer/plugin/host/gaffer/opencode_link-0.3.0.zip new file mode 100644 index 0000000..36e5881 Binary files /dev/null and b/integrations/gaffer/plugin/host/gaffer/opencode_link-0.3.0.zip differ diff --git a/integrations/gaffer/plugin/methods/tools/gaffer_apply.json b/integrations/gaffer/plugin/methods/tools/gaffer_apply.json new file mode 100644 index 0000000..2d7b454 --- /dev/null +++ b/integrations/gaffer/plugin/methods/tools/gaffer_apply.json @@ -0,0 +1,27 @@ +{ + "name": "gaffer_apply", + "domain": "tools", + "description": "A revision-checked graph intent of 1 to 50 typed operations, applied with permission as one undo step (restored on failure): node.create {type or recipe, parent?, name?, bind?}, node.rename {target, name}, plug.set_value {target, value}, graph.connect {source, destination}, graph.connect_many {sources, destination, start_index?}, graph.delete {target}, output.define {target, name, file_name, ...}. render.start, render.cancel and ui.view are side effects, each the sole operation. expect: postconditions. For anything else, gaffer_execute_code.", + "kind": "host-backed", + "risk": "write", + "args": { + "if_revision": { "type": "integer", "required": true, "description": "The revision of your last gaffer_context." }, + "command_id": { + "type": "string", + "required": true, + "description": "Idempotent ID: a replay applies nothing twice." + }, + "summary": { "type": "string" }, + "operations": { + "type": "array", + "items": { "type": "object" }, + "description": "{op, ...} with target, source, destination, sources, start_index, value, type, recipe, variant, name, parent, bind, renderer, file_name, driver, channels, frame_range, parameters as each operation needs. $alias names a node created earlier in the intent." + }, + "expect": { + "type": "array", + "items": { "type": "object" }, + "description": "{subject, predicate: exists, name_equals, value_equals, connected_to, value?}" + } + }, + "execution": { "strategy": "host_method", "method": "apply" } +} diff --git a/integrations/gaffer/plugin/methods/tools/gaffer_context.json b/integrations/gaffer/plugin/methods/tools/gaffer_context.json new file mode 100644 index 0000000..43c3578 --- /dev/null +++ b/integrations/gaffer/plugin/methods/tools/gaffer_context.json @@ -0,0 +1,41 @@ +{ + "name": "gaffer_context", + "domain": "tools", + "description": "Bounded read of the attached Gaffer script: script_handle, revision, the file, selection, focus, and node cards (ref, name, type, plugs; values and connections with include). refs, match (glob on name and type) and radius pick the nodes; without them, the selection, focus and recent nodes. Call first; pass revision to gaffer_apply. quarantined: changes wait until release_quarantine with this read's revision.", + "kind": "host-backed", + "risk": "read", + "args": { + "refs": { "type": "array", "items": { "type": "string" }, "description": "Up to 50 node or plug refs." }, + "match": { "type": "object", "description": "{name, type}: glob patterns." }, + "radius": { "type": "integer", "description": "Neighbours of the picked nodes, 0 to 2.", "default": 0 }, + "plugs": { + "type": "array", + "items": { "type": "string" }, + "description": "Plug names to read on the picked nodes." + }, + "plug_limit": { "type": "integer", "description": "Plugs per node, 1 to 1000." }, + "recipe_match": { + "type": "object", + "description": "{id?, node_type?, menu_path?, variant?}: the node menu's recipes (with include recipes)." + }, + "job": { + "type": "object", + "description": "{id, after?, limit?}: a render job started by gaffer_apply render.start." + }, + "include": { + "type": "array", + "items": { "type": "string" }, + "description": "values, connections, schema, affordances, capabilities, recipes, renders." + }, + "limit": { "type": "integer", "description": "Nodes listed, 1 to 100.", "default": 20 }, + "if_revision": { + "type": "integer", + "description": "The changes since that revision, when the journal still holds them." + }, + "release_quarantine": { + "type": "integer", + "description": "After a failed change that could not be proven undone: the revision of the read that showed the script, once you checked it; lifts the quarantine." + } + }, + "execution": { "strategy": "host_method", "method": "context" } +} diff --git a/integrations/gaffer/plugin/methods/tools/gaffer_execute_code.json b/integrations/gaffer/plugin/methods/tools/gaffer_execute_code.json new file mode 100644 index 0000000..e34326a --- /dev/null +++ b/integrations/gaffer/plugin/methods/tools/gaffer_execute_code.json @@ -0,0 +1,19 @@ +{ + "name": "gaffer_execute_code", + "domain": "tools", + "description": "DANGEROUS: run Python in the attached Gaffer on its script (script, Gaffer, GafferScene, GafferImage, GafferDispatch, IECore, imath), given as code or as a script file (path). Assign a JSON-able dict to `result`; print() output is returned. A run that changes the script is one undo step; a failing one is undone, and a failure says what Gaffer knows (closest plug or class names, the plug a computation failed on, the expected value type). Every run asks permission.", + "kind": "host-backed", + "risk": "destructive", + "dangerous": true, + "args": { + "code": { "type": "string", "description": "Python code (or path)." }, + "path": { + "type": "string", + "description": "A .py file to run instead of code, read as is (absolute or relative to the project): long scripts, no escaping." + }, + "read_only": { "type": "boolean", "description": "Inspection: no undo step." }, + "timeout_s": { "type": "integer", "description": "Longest wait in seconds (1 to 3600).", "default": 120 } + }, + "execution": { "strategy": "host_method", "method": "execute_code" }, + "requires": { "hostMethods": ["execute_code"] } +} diff --git a/integrations/gaffer/plugin/methods/tools/gaffer_hosts.json b/integrations/gaffer/plugin/methods/tools/gaffer_hosts.json new file mode 100644 index 0000000..1e3232a --- /dev/null +++ b/integrations/gaffer/plugin/methods/tools/gaffer_hosts.json @@ -0,0 +1,12 @@ +{ + "name": "gaffer_hosts", + "domain": "tools", + "description": "List the Gaffer scripts offered to OpenCode on this computer (Gaffer with OpenCode Link installed offers each open script when it starts): label, document, Gaffer version, and whether this conversation or another one is attached to each (attached_to: the one this conversation acts on), with the installed part's state. attach: attach one (its offerID) to this conversation, which asks permission; detach: release this conversation's script. With one offered script that no conversation uses, the first Gaffer tool call attaches it.", + "kind": "hostless", + "risk": "write", + "args": { + "attach": { "type": "string", "description": "An offerID from this list." }, + "detach": { "type": "boolean", "description": "Release this conversation's script." } + }, + "execution": { "strategy": "plugin" } +} diff --git a/integrations/gaffer/plugin/methods/tools/gaffer_inspect.json b/integrations/gaffer/plugin/methods/tools/gaffer_inspect.json new file mode 100644 index 0000000..781a8ee --- /dev/null +++ b/integrations/gaffer/plugin/methods/tools/gaffer_inspect.json @@ -0,0 +1,41 @@ +{ + "name": "gaffer_inspect", + "domain": "tools", + "description": "Ask THIS Gaffer what it really exposes, in one call: search (words found in node class names and descriptions, plug names and descriptions of every node class of the loaded modules, and Gaffer, GafferScene, GafferImage and GafferDispatch classes, functions and methods with their signature line; ranked), a node class (description, base classes, every plug with its type, default, range, presets and description, its documentation page), a live node (type, changed plugs with values and input connections, outputs and where they go, compute errors, children; a scene output's bound, sets, globals and locations with object type and attributes; an image output's format, data window, channels and metadata) and/or a Python symbol (its docstring signature). Entries beyond limit are still named. Always says the Gaffer version and the local documentation. Read only.", + "kind": "host-backed", + "risk": "read", + "args": { + "search": { + "type": "string", + "description": "Words, e.g. \"sphere\", \"resolution\", \"set name\"." + }, + "type": { + "type": "string", + "description": "A node class: Module.Class (GafferScene.Sphere, GafferImage.ImageWriter) or a class name." + }, + "node": { + "type": "string", + "description": "A live node path in the script, e.g. Group or Box.sphere." + }, + "gaffer": { + "type": "string", + "description": "A Python path, e.g. Gaffer.GraphComponent.setName, GafferScene.ScenePlug, IECore.CompoundData." + }, + "match": { + "type": "string", + "description": "Keep the plugs (and scene locations) matching: a glob on names (transform.*), or words found in name or description." + }, + "limit": { + "type": "integer", + "description": "Entries in detail (1 to 200); the others are named.", + "default": 40 + } + }, + "execution": { + "strategy": "program", + "program": "programs/inspect.py" + }, + "requires": { + "hostMethods": ["execute_code"] + } +} diff --git a/integrations/gaffer/plugin/methods/tools/gaffer_install_host.json b/integrations/gaffer/plugin/methods/tools/gaffer_install_host.json new file mode 100644 index 0000000..d415317 --- /dev/null +++ b/integrations/gaffer/plugin/methods/tools/gaffer_install_host.json @@ -0,0 +1,9 @@ +{ + "name": "gaffer_install_host", + "domain": "tools", + "description": "Install or update OpenCode Link (the version of this plugin) in Gaffer's user startup folder (~/gaffer/startup/gui) of the computer that runs OpenCode. Asks permission; restart Gaffer afterwards if it is open.", + "kind": "hostless", + "risk": "write", + "args": {}, + "execution": { "strategy": "plugin" } +} diff --git a/integrations/gaffer/plugin/methods/tools/gaffer_render.json b/integrations/gaffer/plugin/methods/tools/gaffer_render.json new file mode 100644 index 0000000..82af2b4 --- /dev/null +++ b/integrations/gaffer/plugin/methods/tools/gaffer_render.json @@ -0,0 +1,58 @@ +{ + "name": "gaffer_render", + "domain": "tools", + "description": "Run a task node (Render, ImageWriter, SceneWriter, SystemCommand...) and its upstream tasks for frames in a separate background gaffer python, dispatched there by a LocalDispatcher: the open Gaffer stays usable and its file keeps its name. It reads a working copy of the open script (default) or the file as saved. The node writes its own outputs, or the EXACT output you name (shots/shot_####.png gives shot_0001.png...; a Render node needs exactly one file output for that), with size [w, h] for a Render node, for this run only. Returns a task: call again with task (and wait_s) for progress, or cancel. Relative paths are next to the .gfr.", + "kind": "host-backed", + "risk": "write", + "args": { + "node": { + "type": "string", + "description": "A task node path, e.g. Render or Box.ImageWriter." + }, + "output": { + "type": "string", + "description": "The exact output path; several frames need a # run or ${frame}." + }, + "frames": { + "type": "any", + "description": "A frame, [start, end] or [start, end, step], whole frames (default: the script's current frame)." + }, + "size": { + "type": "array", + "items": { + "type": "integer" + }, + "description": "Render node only: [width, height], 4 to 16384." + }, + "source": { + "type": "string", + "description": "What is run.", + "enum": ["copy", "saved"], + "default": "copy" + }, + "overwrite": { + "type": "boolean", + "default": false + }, + "task": { + "type": "string", + "description": "A task from an earlier call: its progress." + }, + "wait_s": { + "type": "integer", + "description": "With task: wait up to this long for it to finish (0 to 3600).", + "default": 0 + }, + "cancel": { + "type": "boolean", + "description": "With task: stop it." + } + }, + "execution": { + "strategy": "program", + "program": "programs/render.py" + }, + "requires": { + "hostMethods": ["execute_code"] + } +} diff --git a/integrations/gaffer/plugin/methods/tools/gaffer_save.json b/integrations/gaffer/plugin/methods/tools/gaffer_save.json new file mode 100644 index 0000000..6a38172 --- /dev/null +++ b/integrations/gaffer/plugin/methods/tools/gaffer_save.json @@ -0,0 +1,27 @@ +{ + "name": "gaffer_save", + "domain": "tools", + "description": "Save the open script; with path, save it under that .gfr (as the open file, or a copy with copy true: the open script keeps its name). Relative paths are next to the current file; an existing other file needs overwrite. Says what was written, read from the disk (path, bytes, time), the open file afterwards and whether unsaved changes remain. gaffer_verify with saved reads the file back.", + "kind": "host-backed", + "risk": "write", + "args": { + "path": { + "type": "string" + }, + "copy": { + "type": "boolean", + "default": false + }, + "overwrite": { + "type": "boolean", + "default": false + } + }, + "execution": { + "strategy": "program", + "program": "programs/save.py" + }, + "requires": { + "hostMethods": ["execute_code"] + } +} diff --git a/integrations/gaffer/plugin/methods/tools/gaffer_verify.json b/integrations/gaffer/plugin/methods/tools/gaffer_verify.json new file mode 100644 index 0000000..915f1e0 --- /dev/null +++ b/integrations/gaffer/plugin/methods/tools/gaffer_verify.json @@ -0,0 +1,51 @@ +{ + "name": "gaffer_verify", + "domain": "tools", + "description": "Check a result by measurement. criteria: your own acceptance checks, each measured on computed scenes, images or plug values (at a frame, without moving the script's frame) and passed or failed with the measured value. errors (default on): the outputs of every node under the roots computed, with each compute error, its plug and the node it arises in. render: a low-resolution Cycles image with facts read from its pixels (size, brightness, black, uniform, transparent), made by a separate background gaffer python on a working copy, so a crash there never touches this Gaffer. saved: the same checks on the file as written on disk, read by a background gaffer python; check the delivered file before reporting. Nothing in the open script changes.", + "kind": "host-backed", + "risk": "read", + "args": { + "criteria": { + "type": "array", + "description": "Up to 50 checks: {expr: a read-only Python expression; op: < <= > >= == != approx between in contains; value; tol (approx, default 1e-4); frame (default: the call's); label}. In expr (scene: a node path, its out, or a ScenePlug path; location: a scene path, default /): node(path), plug(path) (its value), scene(path), exists(scene, location), bound(scene, location).min/.max/.size/.center, world_bound(scene, location), translate(scene, location), attributes(scene, location), child_names(scene, location), object(scene, location), object_type(scene, location), primitive_variables(scene, location), set_names(scene), set_members(scene, set_name), globals(scene), image(path) (format, channels, average, min, max), errors(path), script, frame, Gaffer, GafferScene, GafferImage, IECore, imath, math. Sequences compare element by element.", + "items": { + "type": "object" + } + }, + "frame": { + "type": "number", + "description": "Default: the script's current frame." + }, + "errors": { + "type": "boolean", + "description": "Report the nodes whose outputs fail to compute.", + "default": true + }, + "roots": { + "type": "array", + "items": { + "type": "string" + }, + "description": "Nodes (Boxes) the errors are read under (default: the whole script, /)." + }, + "render": { + "type": "any", + "description": "true, or {scene?, camera?, size?: longest side 64 to 1024 (default 320), samples? (default 16), frame?}. Default scene: the input of the first Render node, else the one scene output that leads nowhere; default camera: the scene's render:camera, else its first camera. The script's own outputs are not written." + }, + "saved": { + "type": "any", + "description": "true: read the open script's file as saved on disk (same criteria, errors and render), or a .gfr path. A background gaffer python reads it while this Gaffer stays free; the answer lists its load errors. Past timeout_s it is stopped and the answer says what it was doing." + }, + "timeout_s": { + "type": "integer", + "description": "For render and saved: how long the background gaffer python may take (default 300; 540 with saved)." + } + }, + "execution": { + "strategy": "program", + "program": "programs/verify.py" + }, + "requires": { + "hostMethods": ["execute_code"] + } +} diff --git a/integrations/gaffer/plugin/package.json b/integrations/gaffer/plugin/package.json index 73444c0..53bca73 100644 --- a/integrations/gaffer/plugin/package.json +++ b/integrations/gaffer/plugin/package.json @@ -1,6 +1,6 @@ { "name": "@neuralnomads/opencode-gaffer", - "version": "0.2.0", + "version": "0.3.0", "description": "OpenCode for Gaffer: bounded graph editing through an authenticated local host, with an optional in-app conversation panel.", "license": "MIT", "type": "module", @@ -22,7 +22,8 @@ }, "files": [ "dist", - "skills", + "methods", + "programs", "host", "README.md", "LICENSE" @@ -58,10 +59,10 @@ "host": { "app": "gaffer", "protocol": 1, - "range": "^0.2.0", + "range": "^0.3.0", "appVersions": ">=1.6", - "payload": "host/gaffer/opencode_link-0.2.0.zip", - "sha256": "b704233e3f45b9f7be2894bf878341162018c9c7cb978a578a6b184e63ebc9b6" + "payload": "host/gaffer/opencode_link-0.3.0.zip", + "sha256": "0ad49be2c312d3a1b8fc2980980707cd717c56ad060aaf0bbb98703141d24e5b" } } } diff --git a/integrations/gaffer/plugin/programs/inspect.py b/integrations/gaffer/plugin/programs/inspect.py new file mode 100644 index 0000000..d067c3c --- /dev/null +++ b/integrations/gaffer/plugin/programs/inspect.py @@ -0,0 +1,524 @@ +# SPDX-License-Identifier: MIT +# gaffer_inspect (fixed program, read only): what THIS Gaffer really exposes for a node class, a live node or a +# Python symbol, or where words appear among its node classes, plugs and their descriptions, with the Gaffer +# documentation of this version. Everything is read from the running Gaffer (node classes, plugs, Metadata, +# computed scenes and images); nothing in the script is created or changed. +# PARAMS is prepended by the plugin: {"search"?, "type"?, "node"?, "gaffer"?, "match"?, "limit"?} +# The namespace holds Gaffer, GafferScene, GafferImage, GafferDispatch, IECore, imath, script and application. +import difflib +import fnmatch +import importlib +import inspect as _inspect +import math +import os +import pathlib +import sys +import time +import types + +import Gaffer +import GafferDispatch +import GafferImage +import GafferScene +import IECore +import imath + +P = PARAMS # noqa: F821 +S = script # noqa: F821 +VERSION = Gaffer.About.versionString() +LIMIT = max(1, min(200, int(P.get("limit") or 40))) +MATCH = P.get("match") +NAMES_MAX = 300 +VALUE_CHARS = 300 +LOCATIONS_MAX = 200 +ONLINE = "https://www.gafferhq.org/documentation/%s/" % VERSION +LOCAL = os.path.join(os.environ.get("GAFFER_ROOT", ""), "doc", "gaffer", "html") +HAS_LOCAL = bool(os.environ.get("GAFFER_ROOT")) and os.path.isfile(os.path.join(LOCAL, "index.html")) + + +def ui_descriptions(): + """Plug and node descriptions are registered by the *UI modules: imported here when Gaffer runs without them.""" + try: + import GafferDispatchUI # noqa: F401 + import GafferImageUI # noqa: F401 + import GafferSceneUI # noqa: F401 + import GafferUI # noqa: F401 + return True + except Exception: # noqa: BLE001 + return False + + +DESCRIPTIONS = ui_descriptions() + + +def doc(page): + """A page of this version's documentation: the local copy when Gaffer ships it, else gafferhq.org.""" + local = os.path.join(LOCAL, *page.split("/")) + if HAS_LOCAL and os.path.isfile(local): + return pathlib.Path(local).as_uri() + return ONLINE + page + + +out = {"gaffer": {"version": VERSION, "python": "%d.%d" % sys.version_info[:2], "ui": "GafferUI" in sys.modules, + "descriptions": DESCRIPTIONS}, + "links": {"docs": doc("index.html"), "nodes": doc("Reference/NodeReference/index.html"), + "scripting": doc("WorkingWithThePythonScriptingAPI/index.html"), + "substitutions": doc("Reference/ScriptingReference/StringSubstitutionSyntax/index.html")}} + + +def num(v): + if isinstance(v, float): + return round(v, 6) if math.isfinite(v) else str(v) + return v + + +def plain(v, depth=0): + """A JSON-ready, bounded copy of a plug value, an imath or IECore value.""" + if isinstance(v, (bool, int)) or v is None: + return v + if isinstance(v, float): + return num(v) + if isinstance(v, str): + return v if len(v) <= VALUE_CHARS else v[:VALUE_CHARS] + "... (%d chars)" % len(v) + if isinstance(v, (imath.Box3f, imath.Box3d, imath.Box2i, imath.Box2f)): + return {"min": plain(v.min(), depth + 1), "max": plain(v.max(), depth + 1)} + if isinstance(v, IECore.Data) and hasattr(v, "value"): + return plain(v.value, depth) + if isinstance(v, IECore.CompoundData) or isinstance(v, IECore.CompoundObject): + return {k: plain(v[k], depth + 1) for k in list(v.keys())[:32]} if depth < 2 else repr(v)[:VALUE_CHARS] + if isinstance(v, Gaffer.GraphComponent): + return v.relativeName(S) if S.isAncestorOf(v) else v.fullName() + if depth < 2 and not isinstance(v, (dict, bytes)): + try: + items = list(v) + except Exception: # noqa: BLE001 + return repr(v)[:VALUE_CHARS] + return [plain(x, depth + 1) for x in items[:32]] + (["... (%d)" % len(items)] if len(items) > 32 else []) + return repr(v)[:VALUE_CHARS] + + +def matches(*texts): + """match: a glob on the name when it has * ? or [, otherwise words all found in name or description.""" + if not MATCH: + return True + if any(c in MATCH for c in "*?["): + return fnmatch.fnmatchcase(texts[0], MATCH) + hay = " ".join(t or "" for t in texts).lower() + return all(w in hay for w in MATCH.lower().split()) + + +def described(target): + try: + text = Gaffer.Metadata.value(target, "description") + except Exception: # noqa: BLE001 + return "" + return " ".join(str(text).split()) if text else "" + + +def class_name(cls): + return "%s.%s" % (cls.__module__.split(".")[0], cls.__name__) + + +def type_page(cls): + return doc("Reference/NodeReference/%s/%s.html" % (cls.__module__.split(".")[0], cls.__name__)) + + +def node_modules(): + """The loaded Gaffer modules holding node classes (not the UI, test or private ones).""" + names = sorted(n for n in sys.modules if n.startswith("Gaffer") and "." not in n and not n.endswith(("UI", "Test"))) + return [(n, sys.modules[n]) for n in names if sys.modules[n] is not None] + + +def node_classes(module): + found = [] + for name in dir(module): + cls = getattr(module, name, None) + if _inspect.isclass(cls) and issubclass(cls, Gaffer.Node) and not name.startswith("_") \ + and cls.__module__.split(".")[0] == module.__name__: + found.append(cls) + return found + + +def walk_plugs(parent, prefix=""): + """Every plug under parent, depth first, with its path relative to the node; internal (__) plugs left out.""" + for plug in parent.children(Gaffer.Plug): + name = plug.getName() + if name.startswith("__"): + continue + path = prefix + name + yield path, plug + if len(plug) and not isinstance(plug, (GafferScene.ScenePlug, GafferImage.ImagePlug)): + yield from walk_plugs(plug, path + ".") + + +def presets(plug): + try: + names = list(Gaffer.NodeAlgo.presets(plug)) + except Exception: # noqa: BLE001 + return [] + return names[:40] + + +def plug_template(path, plug): + d = {"name": path, "type": type(plug).__name__} + if plug.direction() == Gaffer.Plug.Direction.Out: + d["direction"] = "out" + if hasattr(plug, "defaultValue") and not isinstance(plug, (GafferScene.ScenePlug, GafferImage.ImagePlug)): + try: + d["default"] = plain(plug.defaultValue()) + except Exception: # noqa: BLE001 + pass + for attr in ("minValue", "maxValue"): + try: + if getattr(plug, "has" + attr[0].upper() + attr[1:])(): + d.setdefault("range", {})[attr[:3]] = plain(getattr(plug, attr)()) + except Exception: # noqa: BLE001 + pass + names = presets(plug) + if names: + d["presets"] = names + text = described(plug) + if text: + d["description"] = text[:240] + return d + + +def detailed(items, info, key, d, name_of): + d[key] = [info(i) for i in items[:LIMIT]] + rest = [name_of(i) for i in items[LIMIT:]] + if rest: + d[key + "_more"] = {"count": len(rest), "names": rest[:NAMES_MAX], "details": "pass match (a glob or words) or a larger limit"} + + +def find_class(name): + """GafferScene.Sphere, GafferScene::Sphere or Sphere (searched in the loaded modules).""" + name = name.replace("::", ".") + if "." in name: + module_name, _, cls_name = name.rpartition(".") + try: + module = sys.modules.get(module_name) or importlib.import_module(module_name) + except Exception: # noqa: BLE001 + return None, [] + cls = getattr(module, cls_name, None) + return (cls if _inspect.isclass(cls) and issubclass(cls, Gaffer.Node) else None), [module] + hits = [c for _, m in node_modules() for c in node_classes(m) if c.__name__ == name] + return (hits[0] if hits else None), hits[1:] + + +def instance(cls): + """A node of the class, made outside the script: node metadata (descriptions) is read through an instance.""" + try: + return cls(), None + except Exception as ex: # noqa: BLE001 + return None, ("%s: %s" % (type(ex).__name__, ex))[:300] + + +def type_info(cls): + node, error = instance(cls) + d = {"type": class_name(cls), "description": described(node) if node is not None else "", "doc": type_page(cls), + "bases": [class_name(b) for b in cls.__mro__[1:] if _inspect.isclass(b) and issubclass(b, Gaffer.Node)][:8]} + if issubclass(cls, GafferDispatch.TaskNode): + d["task"] = True + if node is None: + d["plugs_error"] = error + return d + plugs = [(p, x) for p, x in walk_plugs(node) if matches(p, described(x))] + detailed(plugs, lambda x: plug_template(*x), "plugs", d, lambda x: x[0]) + if MATCH and not plugs: + d["match_found_nothing"] = {"match": MATCH, "plugs": [p for p, _ in walk_plugs(node)][:NAMES_MAX]} + return d + + +if P.get("type"): + wanted = str(P["type"]) + cls, also = find_class(wanted) + if cls is None: + bare = wanted.replace("::", ".").rpartition(".")[2] + everything = {class_name(c): c.__name__ for _, m in node_modules() for c in node_classes(m)} + close = [full for full, short in everything.items() if short in difflib.get_close_matches(bare, list(everything.values()), n=6, cutoff=0.6)] + out["type"] = {"error": "no node class %r in this Gaffer" % wanted, "closest": sorted(close)[:20], + "loaded_modules": [n for n, _ in node_modules()], "details": "pass Module.Class, e.g. GafferScene.Sphere, or search words"} + else: + out["type"] = type_info(cls) + if also and _inspect.isclass(also[0]): + out["type"]["also_in"] = [class_name(c) for c in also] + + +# --- live node ---------------------------------------------------------------------------------------------- +def resolve(path): + """A node or plug of the script: Box.sphere, Box/sphere or /Box/sphere.""" + text = str(path).strip().strip("/").replace("/", ".") + return S.descendant(text) if text else S + + +def plug_entry(path, plug): + d = {"name": path, "type": type(plug).__name__} + source = plug.getInput() + if source is not None: + d["from"] = plain(source) + if isinstance(plug, Gaffer.ValuePlug) and not isinstance(plug, (GafferScene.ScenePlug, GafferImage.ImagePlug, Gaffer.ObjectPlug)) \ + and not len(plug): + try: + d["value"] = plain(plug.getValue()) + except Exception as ex: # noqa: BLE001 + d["error"] = ("%s: %s" % (type(ex).__name__, " ".join(str(ex).split())))[:300] + targets = [o for o in plug.outputs() if o.node() is not None and not o.node().isSame(plug.node())] + if targets: + d["to"] = [plain(o) for o in targets[:12]] + return d + + +def scene_summary(scene): + d = {} + started = time.time() + try: + d["bound"] = plain(scene.bound("/")) + d["sets"] = {} + for name in list(scene.setNames())[:40]: + d["sets"][str(name)] = scene.set(name).value.size() + g = scene.globals() + d["globals"] = sorted(str(k) for k in g.keys())[:60] + listed, total, queue = [], 0, ["/"] + while queue and time.time() - started < 10: + location = queue.pop(0) + names = [str(n) for n in scene.childNames(location)] + for n in names: + child = location.rstrip("/") + "/" + n + total += 1 + if len(listed) < max(LIMIT, LOCATIONS_MAX // 4) and matches(child, ""): + entry = {"path": child} + obj = scene.object(child) + if not isinstance(obj, IECore.NullObject): + entry["object"] = obj.typeName() + attributes = scene.attributes(child) + if len(attributes): + entry["attributes"] = sorted(attributes.keys())[:20] + listed.append(entry) + if total < LOCATIONS_MAX: + queue.append(child) + d["locations"] = listed + d["locations_counted"] = total + if queue: + d["locations_more"] = "not all read: pass match (a glob on location paths) or read the node with gaffer_verify" + except Exception as ex: # noqa: BLE001 + d["error"] = ("%s: %s" % (type(ex).__name__, " ".join(str(ex).split())))[:600] + return d + + +def image_summary(image): + d = {} + try: + fmt = image.format() + d["format"] = [fmt.width(), fmt.height()] + d["data_window"] = plain(image.dataWindow()) + d["channels"] = [str(c) for c in image.channelNames()][:40] + d["metadata"] = sorted(str(k) for k in image.metadata().keys())[:40] + except Exception as ex: # noqa: BLE001 + d["error"] = ("%s: %s" % (type(ex).__name__, " ".join(str(ex).split())))[:600] + return d + + +def node_info(node): + cls = type(node) + d = {"path": "/" if node.isSame(S) else node.relativeName(S), "type": class_name(cls), "description": described(node), + "doc": type_page(cls)} + plugs = [(p, x) for p, x in walk_plugs(node) if matches(p, described(x))] + inputs = [(p, x) for p, x in plugs if x.direction() == Gaffer.Plug.Direction.In] + outputs = [(p, x) for p, x in plugs if x.direction() == Gaffer.Plug.Direction.Out and x.parent().isSame(node)] + changed = [(p, x) for p, x in inputs if x.getInput() is not None or (isinstance(x, Gaffer.ValuePlug) and not len(x) and not x.isSetToDefault())] + changed_paths = {p for p, _ in changed} + default = [(p, x) for p, x in inputs if p not in changed_paths and not len(x)] + detailed(changed, lambda x: plug_entry(*x), "plugs_changed", d, lambda x: x[0]) + if default: + d["plugs_at_default"] = {"count": len(default), "names": [p for p, _ in default][:NAMES_MAX], + "details": "pass match to read them with their values"} + if MATCH: + d["plugs_at_default"]["values"] = [plug_entry(*x) for x in default[:LIMIT]] + d["outputs"] = [plug_entry(*x) for x in outputs[:LIMIT]] + errors = [] + for _, plug in outputs: + try: + if isinstance(plug, GafferScene.ScenePlug): + d.setdefault("scene", {})[plug.getName()] = scene_summary(plug) + if "error" in d["scene"][plug.getName()]: + errors.append({"plug": plug.getName(), "error": d["scene"][plug.getName()]["error"]}) + elif isinstance(plug, GafferImage.ImagePlug): + d.setdefault("image", {})[plug.getName()] = image_summary(plug) + if "error" in d["image"][plug.getName()]: + errors.append({"plug": plug.getName(), "error": d["image"][plug.getName()]["error"]}) + except Exception as ex: # noqa: BLE001 + errors.append({"plug": plug.getName(), "error": ("%s: %s" % (type(ex).__name__, ex))[:600]}) + errors += [{"plug": e["name"], "error": e["error"]} for e in [plug_entry(p, x) for p, x in outputs] if "error" in e] + if errors: + d["errors"] = errors[:10] + if isinstance(node, GafferDispatch.TaskNode): + d["task"] = {"pre_tasks": [plain(p.getInput()) for p in node["preTasks"] if p.getInput() is not None], + **({"fileName": node["fileName"].getValue()} if "fileName" in node else {})} + children = node.children(Gaffer.Node) + if children: + d["children"] = [{"name": c.getName(), "type": class_name(type(c))} for c in children[:LIMIT]] + if len(children) > LIMIT: + d["children_more"] = {"count": len(children) - LIMIT, "names": [c.getName() for c in children[LIMIT:LIMIT + NAMES_MAX]]} + return d + + +if P.get("node"): + path = str(P["node"]) + target = resolve(path) + if isinstance(target, Gaffer.Plug): + target = target.node() + if target is None: + parts = [p for p in path.strip("/").replace("/", ".").split(".") if p] + parent, depth = None, len(parts) + while depth > 0 and parent is None: + depth -= 1 + parent = S.descendant(".".join(parts[:depth])) if depth else S + names = [c.getName() for c in parent.children(Gaffer.Node)] if parent is not None else [] + missing = parts[depth] if depth < len(parts) else "" + out["node"] = {"error": "no node %r in the script" % path, "deepest_existing": parent.relativeName(S) if parent is not None and parent is not S else "", + "closest": difflib.get_close_matches(missing, names, n=6, cutoff=0.5), "children": names[:NAMES_MAX]} + else: + out["node"] = node_info(target) + + +# --- Python symbol ------------------------------------------------------------------------------------------ +def doc_head(obj, lines=40): + text = (getattr(obj, "__doc__", None) or "").strip("\n") + rows = text.splitlines() + return "\n".join(rows[:lines]) + ("\n... (%d more lines)" % (len(rows) - lines) if len(rows) > lines else "") + + +if P.get("gaffer"): + path = str(P["gaffer"]).strip() + parts = path.split(".") + try: + target = sys.modules.get(parts[0]) or importlib.import_module(parts[0]) + except Exception: # noqa: BLE001 + target = None + if target is None or not (parts[0].startswith(("Gaffer", "IECore")) or parts[0] == "imath"): + out["gaffer"] = {"error": "%r is not a Gaffer, IECore or imath module here" % parts[0], + "modules": [n for n, _ in node_modules()] + ["IECore", "IECoreScene", "IECoreImage", "imath"]} + else: + reached, missing = [parts[0]], None + for part in parts[1:]: + nxt = getattr(target, part, None) + if nxt is None: + missing = part + break + target = nxt + reached.append(part) + if missing is not None: + names = [n for n in dir(target) if not n.startswith("_")] + out["gaffer"] = {"error": "%s has no %r" % (".".join(reached), missing), + "closest": [".".join(reached + [n]) for n in difflib.get_close_matches(missing, names, n=8, cutoff=0.5)]} + else: + info = {"path": path, "kind": "class" if _inspect.isclass(target) else "module" if isinstance(target, types.ModuleType) + else "callable" if callable(target) else type(target).__name__, "doc": doc_head(target)} + if _inspect.isclass(target): + own = [n for n in vars(target) if not n.startswith("_")] + info["bases"] = [class_name(b) for b in target.__mro__[1:] if b.__module__.split(".")[0].startswith(("Gaffer", "IECore"))][:12] + info["methods"] = sorted(n for n in own if matches(n, (getattr(getattr(target, n, None), "__doc__", "") or "")[:200]))[:NAMES_MAX] + inherited = sorted(n for n in dir(target) if not n.startswith("_") and n not in own) + if inherited: + info["inherited"] = inherited[:NAMES_MAX] + if issubclass(target, Gaffer.Node): + info["doc_page"] = type_page(target) + node, _ = instance(target) + text = described(node) if node is not None else "" + if text: + info["description"] = text + elif not callable(target) and not isinstance(target, types.ModuleType): + info["value"] = plain(target) + out["gaffer"] = info + + +# --- search ------------------------------------------------------------------------------------------------- +def plug_index(): + """(class, plug path, description) for every plug of every node class of the loaded modules, kept for this + Gaffer session (rebuilt when another module is loaded).""" + cache = sys.modules.get("_opencode_gaffer_inspect_cache") + if cache is None: + cache = types.ModuleType("_opencode_gaffer_inspect_cache") + sys.modules[cache.__name__] = cache + key = (tuple(n for n, _ in node_modules()), DESCRIPTIONS) + if getattr(cache, "key", None) != key: + rows, kinds = [], [] + for _, module in node_modules(): + for cls in node_classes(module): + node, _ = instance(cls) + kinds.append((cls, described(node) if node is not None else "")) + if node is None: + continue + for path, plug in walk_plugs(node): + if path != "user": + rows.append((class_name(cls), path, described(plug))) + cache.key, cache.plugs, cache.kinds = key, rows, kinds + return cache.plugs, cache.kinds + + +def search(text): + words = [w.lower() for w in str(text).split() if w] + if not words: + raise ValueError("search: give one or more words") + started = time.time() + + def score(name, desc): + name = name.lower() + hay = " ".join((name, (desc or "").lower())) + if not all(w in hay for w in words): + return 0 + return sum(3 if w in name else 1 for w in words) + (2 if name in words else 0) + + plugs, kinds = plug_index() + hits = [] + for cls, desc in kinds: + s = score(class_name(cls), desc) + if s: + hits.append((s + 3, {"kind": "node type", "path": class_name(cls), "description": desc[:200], "doc": type_page(cls)})) + by_plug = {} + for owner, path, desc in plugs: + s = score(path.rsplit(".", 1)[-1], desc[:300]) + if s: + # One entry per plug name and meaning: the classes sharing a description are listed together. + entry = by_plug.setdefault((path, desc), [s, []]) + entry[1].append(owner) + for (path, desc), (s, owners) in by_plug.items(): + hits.append((s - 1, {"kind": "plug", "path": path, "description": desc[:200], "on": owners[:8], + **({"on_count": len(owners)} if len(owners) > 8 else {})})) + for module_name in ("Gaffer", "GafferScene", "GafferImage", "GafferDispatch"): + module = sys.modules[module_name] + for name in dir(module): + obj = getattr(module, name, None) + if name.startswith("_") or (_inspect.isclass(obj) and issubclass(obj, Gaffer.Node)): + continue + first = " ".join((getattr(obj, "__doc__", None) or "").split()[:30]) + s = score(name, first) + if s: + hits.append((s - 0.5, {"kind": "class" if _inspect.isclass(obj) else "function" if callable(obj) else "value", + "path": "%s.%s" % (module_name, name), "doc": first[:160]})) + if _inspect.isclass(obj): + for m in vars(obj): + if m.startswith("_"): + continue + mdoc = (getattr(getattr(obj, m, None), "__doc__", None) or "").strip() + s = score(m, " ".join(mdoc.split()[:40])) + if s: + hits.append((s - 0.75, {"kind": "method", "path": "%s.%s.%s" % (module_name, name, m), + "signature": (mdoc.splitlines()[0] if mdoc else "")[:200]})) + hits.sort(key=lambda h: (-h[0], len(h[1]["path"]), h[1]["path"])) + found = {"words": words, "found": len(hits), "results": [h for _, h in hits[:LIMIT]], "seconds": round(time.time() - started, 2), + "modules": [n for n, _ in node_modules()]} + if len(hits) > LIMIT: + found["more"] = {"count": len(hits) - LIMIT, "paths": [h[1]["path"] for h in hits[LIMIT:LIMIT + NAMES_MAX]], + "details": "add words, or inspect a type, node or gaffer path"} + return found + + +if P.get("search"): + out["search"] = search(P["search"]) + +if not any(P.get(k) for k in ("search", "type", "node", "gaffer")): + raise ValueError("give search (words), type (e.g. GafferScene.Sphere), node (a node path, e.g. Group.sphere) " + "or gaffer (e.g. Gaffer.Node.setName)") + +result = out diff --git a/integrations/gaffer/plugin/programs/render.py b/integrations/gaffer/plugin/programs/render.py new file mode 100644 index 0000000..ded55ea --- /dev/null +++ b/integrations/gaffer/plugin/programs/render.py @@ -0,0 +1,202 @@ +# SPDX-License-Identifier: MIT +# gaffer_render (fixed program, runs in the open Gaffer): checks the request, writes a working copy of the open +# script (serialiseToFile: the open file keeps its name and its own file is untouched) or takes the saved file, then +# starts a detached background `gaffer python` running render_job.py on it: the task node (Render, ImageWriter, +# SceneWriter, any TaskNode) and its upstream tasks dispatched by a LocalDispatcher for the frames, to the exact +# output when given. Returns at once; the job writes a log the plugin follows: +# OPENCODE_RENDER {...} what is rendered (once, at the start) +# OPENCODE_PROGRESS {"done": n, "total": m, ...} +# OPENCODE_DONE {"output": ..., "files": [...], ...} or OPENCODE_FAILED "" +# PARAMS: {"node", "output"?, "frames"?, "size"?, "source"?: "copy" | "saved", "overwrite"?} +# JOB: render_job.py, prepended by the plugin. +import json +import os +import re +import subprocess +import tempfile +import time +import uuid + +import Gaffer +import GafferDispatch +import GafferScene +import IECoreScene + +P = PARAMS # noqa: F821 +S = script # noqa: F821 +RUN = re.compile(r"#+") +INTERACTIVE = ("ieDisplay",) + +if not P.get("node"): + raise ValueError("give node: a task node path, e.g. Render or ImageWriter") +current = S["fileName"].getValue() +never_saved = not current +base_dir = os.path.dirname(current) if not never_saved else None +path = str(P["node"]) +node = S.descendant(path.strip().strip("/").replace("/", ".")) +if node is None: + raise ValueError("no node %r in the script" % path) +if not isinstance(node, GafferDispatch.TaskNode): + raise ValueError("node: %s is a %s.%s, not a task node (Render, ImageWriter, SceneWriter, SystemCommand...)" + % (path, type(node).__module__.split(".")[0], type(node).__name__)) +facts = {} + + +def frame_list(spec): + if spec is None: + return None + if isinstance(spec, (int, float)): + values = [spec, spec, 1] + elif isinstance(spec, list) and 2 <= len(spec) <= 3 and all(isinstance(v, (int, float)) for v in spec): + values = list(spec) + ([1] if len(spec) == 2 else []) + else: + raise ValueError("frames: a frame, [start, end] or [start, end, step]") + if not all(float(v).is_integer() for v in values): + raise ValueError("frames: whole frames (Gaffer dispatches integer frame lists)") + start, end, step = (int(v) for v in values) + if end < start or step <= 0: + raise ValueError("frames: end before start, or a step that is not positive") + count = (end - start) // step + 1 + if count > 10000: + raise ValueError("frames: %d frames, at most 10000" % count) + return [start + i * step for i in range(count)] + + +def at_frame(frame): + context = Gaffer.Context(S.context()) + context.setFrame(frame) + return context + + +def file_outputs(render, frame): + """A Render node's file outputs at a frame: {name: Output} (interactive displays left out).""" + with at_frame(frame): + g = render["in"]["globals"].getValue() + found = {} + for key in g.keys(): + value = g[key] + if key.startswith("output:") and isinstance(value, IECoreScene.Output) and value.getType() not in INTERACTIVE: + found[key[len("output:"):]] = value + return found + + +frames = frame_list(P.get("frames")) +if frames is None: + now = S.context().getFrame() + frames = [int(round(now))] +rendering = isinstance(node, GafferScene.Render) and node["mode"].getValue() == GafferScene.Render.Mode.RenderMode +out_kind = out_name = output = output_pattern = None +if rendering: + outputs = file_outputs(node, frames[0]) + with at_frame(frames[0]): + facts["renderer"] = node["resolvedRenderer"].getValue() or None + if not facts["renderer"]: + facts["renderer_note"] = "no renderer: set the Render node's renderer, or render:defaultRenderer in StandardOptions" + if len(outputs) == 1: + out_name = next(iter(outputs)) + out_kind = "output" + patterns = {name: o.getName() for name, o in outputs.items()} + facts["outputs"] = patterns +elif "fileName" in node and isinstance(node["fileName"], Gaffer.StringPlug): + out_kind = "fileName" +if P.get("output"): + output = P["output"] + if not os.path.isabs(output): + if base_dir is None: + raise ValueError("output %r is relative and the script was never saved: give an absolute path" % output) + output = os.path.join(base_dir, output) + output = os.path.normpath(output).replace("\\", "/") + runs = RUN.findall(os.path.basename(output)) + if len(runs) > 1: + raise ValueError("output: one run of # marks the frame number") + if len(frames) > 1 and not runs and "${frame}" not in output and "$frame" not in output: + raise ValueError("output: %d frames need a frame number in the name: a # run (shot_####.exr) or ${frame}" % len(frames)) + if out_kind is None: + raise ValueError("node: %s has %s: name the output in the script" % ( + path, "%d file outputs" % len(facts.get("outputs", {})) if rendering else "no fileName plug")) + output_pattern = output + + +def targets_at(frame): + with at_frame(frame) as context: + if output_pattern: + names = [context.substitute(output_pattern)] + elif rendering: + names = [o.getName() for o in file_outputs(node, frame).values()] + elif out_kind == "fileName": + names = [node["fileName"].getValue()] + else: + names = [] + # ImageWriter and SceneWriter keep # in their fileName for sequences: the frame is put in here. + return [os.path.abspath(context.substitute(n)) for n in names if n] + + +targets = [t for f in frames for t in targets_at(f)] +output = output_pattern or (node["fileName"].getValue() if out_kind == "fileName" else + next(iter(facts["outputs"].values())) if rendering and len(facts.get("outputs", {})) == 1 else None) +if not targets: + facts["targets_note"] = "the node names no output file: what it wrote is read back from the log" +taken = [t for t in targets if os.path.exists(t)] +if taken and not P.get("overwrite"): + raise ValueError("%s already exists (%d file(s)): pass overwrite true or name another output" % (taken[0], len(taken))) +for folder in sorted({os.path.dirname(t) for t in targets})[:100]: + os.makedirs(folder, exist_ok=True) +size = P.get("size") +if size is not None: + if not isinstance(size, list) or len(size) != 2 or not all(isinstance(v, int) and 4 <= v <= 16384 for v in size): + raise ValueError("size: [width, height], 4 to 16384") + if not rendering: + raise ValueError("size: a Render node's (in render mode); %s renders no image itself" % path) + +source = P.get("source", "copy") +if source not in ("copy", "saved"): + raise ValueError("source: copy (a working copy of the open script, the default) or saved (the file as saved)") +stamp = time.strftime("%Y%m%d-%H%M%S") +# One work folder per job (several can start in the same second from the same Gaffer): never shared. +work = os.path.join(tempfile.gettempdir(), "opencode-gaffer-render", "%s-%d-%s" % (stamp, os.getpid(), uuid.uuid4().hex[:12])) +os.makedirs(work) +working_copy = None +if source == "saved": + if never_saved or not os.path.isfile(current): + raise ValueError("source saved: the script was never saved (gaffer_save first, or source copy)") + gfr = current + if S["unsavedChanges"].getValue(): + facts["unsaved_changes"] = "the open script has changes the saved file lacks: they are not rendered" +else: + # The copy keeps the script's file name, so ${script:name} expands as in the open script. + working_copy = os.path.join(work, os.path.basename(current) or "untitled.gfr").replace("\\", "/") + S.serialiseToFile(working_copy) + if not os.path.isfile(working_copy): + raise RuntimeError("the working copy was not written") + gfr = working_copy + +job = {"file": gfr, "node": node.relativeName(S), "frames": frames, "targets": targets, "out_kind": out_kind, + "out_name": out_name, "output_pattern": output_pattern, "size": size, "work": work} +with open(os.path.join(work, "job.json"), "w", encoding="utf-8") as fh: + json.dump(job, fh) +job_script = os.path.join(work, "render_job.py") +with open(job_script, "w", encoding="utf-8") as fh: + fh.write(JOB) # noqa: F821 - render_job.py, prepended by the plugin +exe = os.path.join(os.environ.get("GAFFER_ROOT") or "", "bin", "gaffer.cmd" if os.name == "nt" else "gaffer") +if not os.path.isfile(exe): + raise RuntimeError("no Gaffer launcher at %s" % exe) +log_path = os.path.join(work, "render.log") +env = {k: v for k, v in os.environ.items() if k != "OPENCODE_GAFFER_OFFER_KEY"} +env["OPENCODE_GAFFER_ROLE"] = "render" # never offered to OpenCode +env["PYTHONUNBUFFERED"] = "1" +log = open(log_path, "wb") +# The open Gaffer's working folder: relative file names resolve as they would there. +kwargs = {"stdin": subprocess.DEVNULL, "stdout": log, "stderr": subprocess.STDOUT, "close_fds": True, "env": env, "cwd": os.getcwd()} +job_json = os.path.join(work, "job.json") +if os.name == "nt": + kwargs["creationflags"] = 0x08000000 | 0x00000200 # no console window, own process group + command = '"%s" /d /s /c ""%s" python "%s" "%s""' % (os.environ.get("ComSpec") or "cmd.exe", exe, job_script, job_json) +else: + kwargs["start_new_session"] = True + command = [exe, "python", job_script, job_json] +process = subprocess.Popen(command, **kwargs) +log.close() +result = {"pid": process.pid, "log": log_path, "work": work, "kind": "render", "output": output, "targets": targets, + "frames": frames, "start": frames[0], "end": frames[-1], "node": node.relativeName(S), + "node_type": "%s.%s" % (type(node).__module__.split(".")[0], type(node).__name__), "source": source, "file": gfr, + "facts": facts, **({"working_copy": working_copy} if working_copy else {})} diff --git a/integrations/gaffer/plugin/programs/render_job.py b/integrations/gaffer/plugin/programs/render_job.py new file mode 100644 index 0000000..bdff4da --- /dev/null +++ b/integrations/gaffer/plugin/programs/render_job.py @@ -0,0 +1,160 @@ +# SPDX-License-Identifier: MIT +# The background job (written next to its log by render.py, run as: gaffer python render_job.py job.json): loads +# the working copy (or saved file) into a fresh ScriptNode, overrides the output and size in memory only, then +# dispatches the task node and its upstream tasks for the frames with a LocalDispatcher in this process, and prints +# for the plugin: +# OPENCODE_RENDER {...}, OPENCODE_PROGRESS {"done": n, "total": m, ...}, then OPENCODE_DONE {...} or +# OPENCODE_FAILED "". +import json +import os +import sys +import threading +import time +import traceback + +import Gaffer +import GafferDispatch +import GafferImage +import GafferScene +import IECore +import IECoreScene +import imath + +with open(sys.argv[1] if len(sys.argv) > 1 else os.path.join(os.path.dirname(sys.argv[0]), "job.json"), encoding="utf-8") as fh: + o = json.load(fh) + + +def mark(key, value): + sys.stdout.write("OPENCODE_%s %s\n" % (key, json.dumps(value))) + sys.stdout.flush() + + +def at_frame(script, frame): + context = Gaffer.Context(script.context()) + context.setFrame(frame) + return context + + +def insert_before(script, node, name, processor): + """processor goes between the Render node and its input scene, in this process only.""" + script[name] = processor + processor["in"].setInput(node["in"].getInput()) + node["in"].setInput(processor["out"]) + return processor + + +def file_outputs(script, node, frame): + with at_frame(script, frame): + g = node["in"]["globals"].getValue() + return {k[len("output:"):]: g[k] for k in g.keys() + if k.startswith("output:") and isinstance(g[k], IECoreScene.Output) and g[k].getType() != "ieDisplay"} + + +def render(): + script = Gaffer.ScriptNode() + script["fileName"].setValue(o["file"].replace("\\", "/")) + with IECore.CapturingMessageHandler() as captured: + script.load(continueOnError=True) + load_warnings = ["%s: %s" % (m.context, m.message) for m in captured.messages if m.level <= IECore.Msg.Level.Warning][:10] + node = script.descendant(o["node"]) + if node is None: + raise RuntimeError("the file read has no node %s" % o["node"]) + rendering = isinstance(node, GafferScene.Render) and node["mode"].getValue() == GafferScene.Render.Mode.RenderMode + used = {"node": o["node"], "type": "%s.%s" % (type(node).__module__.split(".")[0], type(node).__name__), + "frames": [o["frames"][0], o["frames"][-1]], "count": len(o["frames"]), + # This process itself: on Windows the task's pid is the cmd.exe that started the launcher. + "pid": os.getpid()} + if load_warnings: + used["load_warnings"] = [w[:300] for w in load_warnings] + if o["output_pattern"]: + if o["out_kind"] == "fileName": + node["fileName"].setValue(o["output_pattern"]) + else: + original = file_outputs(script, node, o["frames"][0])[o["out_name"]] + drop = insert_before(script, node, "opencodeRenderNoOutput", GafferScene.DeleteOutputs()) + drop["names"].setValue(o["out_name"]) + add = insert_before(script, node, "opencodeRenderOutput", GafferScene.Outputs()) + add.addOutput(o["out_name"], IECoreScene.Output(o["output_pattern"], original.getType(), original.getData(), original.parameters())) + if o["size"]: + options = insert_before(script, node, "opencodeRenderSize", GafferScene.StandardOptions()) + for name, value in (("render:resolution", imath.V2i(*o["size"])), ("render:resolutionMultiplier", 1.0)): + options["options"][name]["enabled"].setValue(True) + options["options"][name]["value"].setValue(value) + used["size"] = o["size"] + targets = [] + for frame in o["frames"]: + with at_frame(script, frame) as context: + if rendering: + names = [out.getName() for out in file_outputs(script, node, frame).values()] + elif "fileName" in node and isinstance(node["fileName"], Gaffer.StringPlug): + names = [node["fileName"].getValue()] + else: + names = [] + targets += [os.path.abspath(context.substitute(n)) for n in names if n] + if rendering: + with at_frame(script, o["frames"][0]): + used["renderer"] = node["resolvedRenderer"].getValue() + if o["targets"] and targets != o["targets"]: + used["targets_differ"] = {"expected": o["targets"][:3], "here": targets[:3]} + used["targets"] = targets[:3] + (["... (%d)" % len(targets)] if len(targets) > 3 else []) + mark("RENDER", used) + total = len(o["frames"]) + state = {"done": 0} + lock = threading.Lock() + + def progress(count): + with lock: + if count > state["done"]: + state["done"] = min(count, total) + mark("PROGRESS", {"done": state["done"], "total": total}) + + stop = threading.Event() + begun = time.time() + per_frame = max(1, len(targets) // total) if targets else 1 + + def watch(): + # The files say how far it is: a LocalDispatcher in this process reports no frame events. + while not stop.wait(1.0): + written = sum(1 for t in targets if os.path.isfile(t) and os.path.getmtime(t) >= begun - 1) + progress(written // per_frame) + + dispatcher = GafferDispatch.LocalDispatcher() + dispatcher["jobsDirectory"].setValue(os.path.join(o["work"], "jobs").replace("\\", "/")) + dispatcher["framesMode"].setValue(GafferDispatch.Dispatcher.FramesMode.CustomRange) + dispatcher["frameRange"].setValue(",".join(str(f) for f in o["frames"])) + dispatcher["executeInBackground"].setValue(False) + watcher = threading.Thread(target=watch, daemon=True) + watcher.start() + try: + with at_frame(script, o["frames"][0]): + dispatcher.dispatch([node]) + finally: + stop.set() + watcher.join(2) + missing = [t for t in targets if not os.path.isfile(t)] + if missing: + raise RuntimeError("%d of %d file(s) were not written, first %s" % (len(missing), len(targets), missing[0])) + progress(total) + done = {"output": o["output_pattern"] or (targets[0] if len(targets) == 1 else None), "files": targets, + "bytes": sum(os.path.getsize(t) for t in targets)} + try: + reader = GafferImage.ImageReader() + reader["fileName"].setValue(targets[0].replace("\\", "/")) + fmt = reader["out"].format() + done["image_size"] = [fmt.width(), fmt.height()] + if o["size"] and done["image_size"] != list(o["size"]): + done["size_note"] = "asked %dx%d, written %dx%d" % (o["size"][0], o["size"][1], fmt.width(), fmt.height()) + except Exception: # noqa: BLE001 - not an image (a scene, USD, a text file) or no file named + pass + mark("DONE", done) + + +code = 0 +try: + render() +except Exception as ex: # noqa: BLE001 + mark("FAILED", ("%s: %s" % (type(ex).__name__, ex))[:1500] + "\n" + traceback.format_exc()[-1500:]) + code = 1 +sys.stdout.flush() +# Leave at once: the outcome is written, nothing in the exit handlers can lose it. +os._exit(code) diff --git a/integrations/gaffer/plugin/programs/save.py b/integrations/gaffer/plugin/programs/save.py new file mode 100644 index 0000000..ee5770a --- /dev/null +++ b/integrations/gaffer/plugin/programs/save.py @@ -0,0 +1,55 @@ +# SPDX-License-Identifier: MIT +# gaffer_save (fixed program): save the open script, or save it under a path (as the open file: its fileName +# changes, or as a copy written with serialiseToFile that leaves the open file's name and its own file alone). Says +# what was really written (path, bytes, time), read from the disk. +# PARAMS: {"path"?, "copy"?, "overwrite"?} +# The namespace holds Gaffer, script and application. +import os +import time + +P = PARAMS # noqa: F821 +S = script # noqa: F821 +EXTENSIONS = (".gfr",) +current = S["fileName"].getValue() +never_saved = not current +target = P.get("path") +copy = bool(P.get("copy")) + +if copy and not target: + raise ValueError("copy: give path, the file the copy goes to") +if target: + if not os.path.isabs(target): + if never_saved: + raise ValueError("path %r is relative and the script was never saved: give an absolute path" % target) + target = os.path.join(os.path.dirname(current), target) + target = os.path.normpath(target) + if os.path.splitext(target)[1].lower() not in EXTENSIONS: + raise ValueError("path must end with .gfr") + same = bool(current) and os.path.normcase(target) == os.path.normcase(os.path.normpath(current)) + if os.path.exists(target) and not same and not P.get("overwrite"): + raise ValueError("%s exists: pass overwrite true or name another file" % target) + os.makedirs(os.path.dirname(target), exist_ok=True) +elif never_saved: + raise ValueError("the script was never saved: pass path") + +written = os.path.normpath(target or current) +before = os.stat(written).st_mtime_ns if os.path.exists(written) else None +if copy: + S.serialiseToFile(written) +else: + if target: + # Gaffer keeps file names with forward slashes. + S["fileName"].setValue(written.replace("\\", "/")) + S.save() +if not os.path.isfile(written): + raise RuntimeError("nothing was written at %s" % written) +stat = os.stat(written) +if before is not None and stat.st_mtime_ns == before: + raise RuntimeError("%s was not rewritten (its modification time did not change)" % written) +result = {"saved": written, "bytes": stat.st_size, + "saved_at": time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(stat.st_mtime)), + "open_file": os.path.normpath(S["fileName"].getValue()) if S["fileName"].getValue() else None, + "unsaved_changes": S["unsavedChanges"].getValue()} +if copy: + result["copy"] = True + result["note"] = "a copy: the open script keeps its name, and its own file is unchanged" diff --git a/integrations/gaffer/plugin/programs/verify.py b/integrations/gaffer/plugin/programs/verify.py new file mode 100644 index 0000000..bad3ec5 --- /dev/null +++ b/integrations/gaffer/plugin/programs/verify.py @@ -0,0 +1,610 @@ +# SPDX-License-Identifier: MIT +# gaffer_verify (fixed program): check a result by measurement on the open script. criteria: the agent's own +# acceptance checks, read-only expressions measured on computed scenes, images and plug values (at a frame, through +# a Gaffer.Context: the script's frame does not move), each passed or failed with its measured value. errors: the +# outputs of every node under the roots computed (scene hierarchy bounded, images' format and a tile, values), with +# the Gaffer.ProcessException each raised and the plug it arose in. render: a low-resolution Cycles image made by a +# separate background `gaffer python` on a working copy (a crash there never touches this Gaffer), with facts read +# from its pixels. Nothing in the open script is changed. +# PARAMS is prepended by the plugin: {"criteria"?, "frame"?, "errors"?, "roots"?, "render"?, "timeout_s"?} +# The namespace holds Gaffer, GafferScene, GafferImage, GafferDispatch, IECore, imath, script and application. +import ast +import json +import math +import os +import re +import subprocess +import tempfile +import time + +import Gaffer +import GafferDispatch +import GafferImage +import GafferScene +import IECore +import imath + +P = PARAMS # noqa: F821 +S = script # noqa: F821 +STARTED = time.time() +CRITERIA = P.get("criteria") or [] +FRAME = float(P["frame"]) if P.get("frame") is not None else S.context().getFrame() +COMPUTE_BUDGET = 60.0 # seconds spent computing outputs for the error scan, at most +ERRORS_MAX = 60 +NODES_MAX = 20000 +LOCATIONS_MAX = 2000 # scene locations read per scene output in the error scan +OPS = ("<", "<=", ">", ">=", "==", "!=", "approx", "between", "in", "contains") +THROWAWAY = os.environ.get("OPENCODE_GAFFER_ROLE") in ("verify", "render") +# gaffer_verify saved: the background job defines STEP, a line in its log saying what it does now (the plugin reads +# it to say where a verify that ran out of time stopped). In the open script: nothing. +STEP = globals().get("STEP") or (lambda **what: None) +if not isinstance(CRITERIA, list) or len(CRITERIA) > 50: + raise ValueError("criteria: a list of at most 50 {expr, op, value, tol?, frame?, label?}") + +SAFE_BUILTINS = {n: __builtins__[n] if isinstance(__builtins__, dict) else getattr(__builtins__, n) for n in ( + "abs", "min", "max", "len", "round", "sum", "any", "all", "sorted", "int", "float", "bool", "str", "list", + "tuple", "set", "dict", "range", "zip", "enumerate", "isinstance", "map", "filter", "reversed")} +# Attribute names that change something (Gaffer, IECore, imath): refused in a criterion. +WRITES = re.compile(r"^(set|create|destroy|delete|remove|insert|append|add|clear|save|load|merge|render|press|move|" + r"copy|paste|lock|unlock|allow|bypass|execute|run|revert|undo|redo|define|override|export|" + r"reload|mute|unmute|change|rename|layout|cut|write|open|close|install|uninstall|exit|system|" + r"dirty|force|update|apply|bake|expand|collapse|hide|show|display|dispatch|serialise|serialize|" + r"connect|disconnect|import|acquire|release|register|deregister|emit|push|pop)(?![a-z])") +READS = {"set", "setNames", "setHash", "displayWindow", "displayWindowHash"} +BLOCKED = {"application", "undoStack", "selection", "focus", "system", "exit"} + + +def read_only_expression(text): + """An expression that reads: no assignment, no lambda, no private name, no attribute that changes things.""" + tree = ast.parse(text, mode="eval") + for n in ast.walk(tree): + if isinstance(n, (ast.NamedExpr, ast.Lambda)): + raise ValueError("criteria: %r is not a plain read" % text) + if isinstance(n, ast.Attribute) and (n.attr.startswith("_") or n.attr in BLOCKED or (WRITES.match(n.attr[:1].lower() + n.attr[1:]) and n.attr not in READS)): + raise ValueError("criteria: %r reaches %s, which is not a read" % (text, n.attr)) + if isinstance(n, ast.Name) and (n.id.startswith("_") or n.id in BLOCKED): + raise ValueError("criteria: %r names %s" % (text, n.id)) + return compile(tree, "", "eval") + + +class Box(object): + """Bounds: min, max, size, center (x, y, z tuples).""" + + def __init__(self, box): + self.min = tuple(box.min()) + self.max = tuple(box.max()) + self.size = tuple(box.size()) + self.center = tuple(box.center()) + + def __repr__(self): + return "Box(min=%s, max=%s)" % (rounded(self.min), rounded(self.max)) + + +def resolve(path): + text = str(path).strip().strip("/").replace("/", ".") + return S.descendant(text) if text else None + + +def need(path): + n = resolve(path) + if n is None: + raise ValueError("no node or plug %r in the script" % path) + return n + + +def node_of(path): + n = need(path) + if not isinstance(n, Gaffer.Node): + raise ValueError("%s is a %s, not a node" % (path, type(n).__name__)) + return n + + +def scene_of(target): + """A ScenePlug: the plug itself, a node's out, or a Render node's in.""" + if isinstance(target, GafferScene.ScenePlug): + return target + n = need(target) if isinstance(target, str) else target + if isinstance(n, GafferScene.ScenePlug): + return n + for name in ("out", "in"): + if isinstance(n, Gaffer.Node) and name in n and isinstance(n[name], GafferScene.ScenePlug): + return n[name] + raise ValueError("%s has no scene output" % (target if isinstance(target, str) else target.relativeName(S))) + + +def image_of(target): + n = need(target) if isinstance(target, str) else target + if isinstance(n, GafferImage.ImagePlug): + return n + for name in ("out", "in"): + if isinstance(n, Gaffer.Node) and name in n and isinstance(n[name], GafferImage.ImagePlug): + return n[name] + raise ValueError("%s has no image output" % (target if isinstance(target, str) else target.relativeName(S))) + + +def plain_data(v): + if isinstance(v, IECore.Data) and hasattr(v, "value"): + return v.value + if isinstance(v, (IECore.CompoundData, IECore.CompoundObject)): + return {str(k): plain_data(v[k]) for k in v.keys()} + return v + + +def world_bound(scene, location): + box, m = scene.bound(location), scene.fullTransform(location) + if box.isEmpty(): + return Box(box) + lo, hi = box.min(), box.max() + corners = [imath.V3f(x, y, z) * m for x in (lo.x, hi.x) for y in (lo.y, hi.y) for z in (lo.z, hi.z)] + world = imath.Box3f() + for c in corners: + world.extendBy(c) + return Box(world) + + +def image_stats(target): + image = image_of(target) + fmt = image.format() + stats = GafferImage.ImageStats() + stats["in"].setInput(image) + stats["area"].setValue(fmt.getDisplayWindow()) + return {"format": (fmt.width(), fmt.height()), "channels": [str(c) for c in image.channelNames()], + "average": tuple(stats["average"].getValue()), "min": tuple(stats["min"].getValue()), "max": tuple(stats["max"].getValue())} + + +def namespace(frame): + def plug(path): + p = need(path) + if not isinstance(p, Gaffer.ValuePlug): + raise ValueError("%s is a %s, not a plug with a value" % (path, type(p).__name__)) + value = p.getValue() + return Box(value) if isinstance(value, (imath.Box3f, imath.Box2f, imath.Box2i)) else plain_data(value) + + def located(fn): + return lambda scene, location="/": fn(scene_of(scene), location) + + return {"__builtins__": SAFE_BUILTINS, "Gaffer": Gaffer, "GafferScene": GafferScene, "GafferImage": GafferImage, + "IECore": IECore, "imath": imath, "math": math, "frame": frame, "script": S, "node": node_of, "plug": plug, + "scene": scene_of, "exists": located(lambda s, l: s.exists(l)), "bound": located(lambda s, l: Box(s.bound(l))), + "world_bound": located(world_bound), "translate": located(lambda s, l: tuple(s.fullTransform(l).translation())), + "attributes": located(lambda s, l: plain_data(s.attributes(l))), + "child_names": located(lambda s, l: [str(n) for n in s.childNames(l)]), + "object": located(lambda s, l: s.object(l)), "object_type": located(lambda s, l: s.object(l).typeName()), + "primitive_variables": located(lambda s, l: sorted(s.object(l).keys()) if hasattr(s.object(l), "keys") else []), + "set_names": lambda scene: sorted(str(n) for n in scene_of(scene).setNames()), + "set_members": lambda scene, name: sorted(scene_of(scene).set(name).value.paths()), + "globals": lambda scene: plain_data(scene_of(scene).globals()), "image": image_stats, + "errors": lambda path: [e["error"] for e in node_errors(node_of(path))]} + + +def rounded(value): + if isinstance(value, float): + return round(value, 6) if math.isfinite(value) else str(value) + if isinstance(value, (bool, int, str)) or value is None: + return value if not isinstance(value, str) or len(value) <= 300 else value[:300] + "..." + if isinstance(value, Box): + return {"min": rounded(value.min), "max": rounded(value.max), "size": rounded(value.size)} + if isinstance(value, (imath.Box3f, imath.Box2f, imath.Box2i)): + return rounded(Box(value)) + if isinstance(value, Gaffer.GraphComponent): + return value.relativeName(S) if S.isAncestorOf(value) else value.fullName() + if isinstance(value, IECore.Object): + data = plain_data(value) + return rounded(data) if data is not value else value.typeName() + if isinstance(value, dict): + return {str(k): rounded(v) for k, v in list(value.items())[:64]} + try: + return [rounded(v) for v in list(value)[:64]] + except TypeError: + return repr(value)[:200] + + +def compare(measured, op, value, tol): + def numbers(x): + if isinstance(x, (str, Box, dict)): + return None + try: + return [float(v) for v in x] + except (TypeError, ValueError): + return None + + if op == "in": + return measured in value + if op == "contains": + return value in measured + if op == "between": + lo, hi = value + seq = numbers(measured) + return all(lo <= v <= hi for v in seq) if seq is not None else lo <= measured <= hi + seq = numbers(measured) + if seq is not None and (op not in ("==", "!=") or isinstance(value, (list, tuple))): + targets = numbers(value) if isinstance(value, (list, tuple)) else [value] * len(seq) + if targets is None or len(targets) != len(seq): + raise ValueError("value: %d numbers to compare with %s" % (len(seq), rounded(measured))) + if op == "!=": + return any(m != t for m, t in zip(seq, targets)) + return all(compare(m, op, t, tol) for m, t in zip(seq, targets)) + if op == "approx": + return abs(measured - value) <= tol + return {"<": lambda: measured < value, "<=": lambda: measured <= value, ">": lambda: measured > value, + ">=": lambda: measured >= value, "==": lambda: measured == value, "!=": lambda: measured != value}[op]() + + +def at_frame(frame): + context = Gaffer.Context(S.context()) + context.setFrame(frame) + return context + + +# --- the error scan: compute every output, report the ProcessExceptions ------------------------------------------ +def error_text(ex): + """A ProcessException's text without its Python traceback: the plug, then the exception that ended it.""" + text = str(ex) + if "Traceback (most recent call last)" in text: + head = text.split(":", 1)[0].strip() + last = [line for line in text.splitlines() if line.strip()][-1].strip() + text = "%s : %s" % (head, last) + return " ".join(text.split())[:400] + + +def compute(plug): + if isinstance(plug, GafferScene.ScenePlug): + plug.globals() + names = list(plug.setNames()) + for name in names[:40]: + plug.set(name) + queue, read = ["/"], 0 + while queue and read < LOCATIONS_MAX: + location = queue.pop(0) + read += 1 + plug.bound(location) + if location != "/": + plug.transform(location) + plug.attributes(location) + plug.object(location) + queue += [location.rstrip("/") + "/" + str(n) for n in plug.childNames(location)] + return + if isinstance(plug, GafferImage.ImagePlug): + plug.format() + window = plug.dataWindow() + channels = plug.channelNames() + plug.metadata() + if channels and not GafferImage.BufferAlgo.empty(window): + plug.channelData(channels[0], GafferImage.ImagePlug.tileOrigin(window.min())) + return + if isinstance(plug, GafferDispatch.TaskNode.TaskPlug): + return + # A compound ValuePlug has no value of its own: its children do. + if isinstance(plug, Gaffer.ValuePlug) and hasattr(plug, "getValue"): + plug.getValue() + return + for child in plug.children(Gaffer.Plug): + compute(child) + + +def output_plugs(node): + # A filter's output is only computed for a scene location, inside the node it filters. + return [p for p in node.children(Gaffer.Plug) if p.direction() == Gaffer.Plug.Direction.Out + and not p.getName().startswith("__") and not isinstance(p, GafferScene.FilterPlug)] + + +NEEDS_CONTEXT = [] # outputs computed only within a context the scan does not make (a loop's, a filter's) + + +def node_errors(node): + found = [] + for plug in output_plugs(node): + try: + compute(plug) + except Exception as ex: # noqa: BLE001 + text = error_text(ex) + if "Context has no variable named" in text: + NEEDS_CONTEXT.append(plug.relativeName(S)) + continue + entry = {"plug": plug.relativeName(S), "error": text} + origin = ex.plug() if isinstance(ex, Gaffer.ProcessException) and hasattr(ex, "plug") else None + if origin is not None: + entry["arose_in"] = origin.relativeName(S) if S.isAncestorOf(origin) else origin.fullName() + entry["origin_node"] = origin.node().relativeName(S) if origin.node() is not None and S.isAncestorOf(origin.node()) else None + found.append(entry) + return found + + +def all_nodes(root): + # A node's private internals ("__" names, such as an ImageWriter's __contextQuery) are its own: its outputs say + # what they lead to. + stack = [root] + while stack: + n = stack.pop(0) + yield n + stack += [c for c in n.children(Gaffer.Node) if not c.getName().startswith("__")] + + +def scan_errors(): + found, computed, skipped, origins = [], 0, [], [] + with_errors = total = 0 + for root in roots: + for n in all_nodes(root): + if n.isSame(S): + continue + total += 1 + if total > NODES_MAX: + break + if time.time() - STARTED > COMPUTE_BUDGET: + skipped.append(n.relativeName(S)) + continue + STEP(step="cook", node=n.relativeName(S)) # the plugin's step name: "computing for the error scan" + computed += len(output_plugs(n)) + errors = node_errors(n) + if not errors: + continue + with_errors += 1 + name = n.relativeName(S) + for e in errors: + origin = e.pop("origin_node", None) or name + if origin not in origins: + origins.append(origin) + here = any(e.get("arose_in", e["plug"]).startswith(name + ".") for e in errors) + if len(found) < ERRORS_MAX: + found.append({"node": name, "type": "%s.%s" % (type(n).__module__.split(".")[0], type(n).__name__), + "errors": errors[:4], "arises_here": here}) + # Where an error arises first, then the nodes it reaches downstream. + found.sort(key=lambda e: 0 if e["arises_here"] else 1) + return found, {"nodes_with_errors": with_errors, "errors_arise_in": origins[:40], "nodes_read": min(total, NODES_MAX), + "outputs_computed": computed, + **({"outputs_needing_context": NEEDS_CONTEXT[:20]} if NEEDS_CONTEXT else {}), + **({"not_computed": skipped[:40], "not_computed_count": len(skipped)} if skipped else {})} + + +rows = [] +for i, c in enumerate(CRITERIA): + if not isinstance(c, dict) or "expr" not in c or c.get("op") not in OPS or "value" not in c: + raise ValueError("criteria[%d]: {expr, op: one of %s, value, tol?, frame?, label?}" % (i, " ".join(OPS))) + code = read_only_expression(str(c["expr"])) + frame = float(c["frame"]) if c.get("frame") is not None else FRAME + row = {"label": c.get("label") or c["expr"], "check": "%s %s %s" % (c["expr"], c["op"], json.dumps(c["value"])), "frame": rounded(frame)} + STEP(step="criterion", index=i + 1, of=len(CRITERIA), label=row["label"][:120], frame=rounded(frame)) + measuring = time.time() + try: + with at_frame(frame): + measured = eval(code, namespace(frame)) # noqa: S307 - a checked read-only expression + row["measured"] = rounded(measured) + row["passed"] = bool(compare(measured, c["op"], c["value"], float(c.get("tol", 1e-4)))) + except Exception as ex: # noqa: BLE001 + row["passed"] = False + row["error"] = ("%s: %s" % (type(ex).__name__, error_text(ex)))[:400] + row["seconds"] = round(time.time() - measuring, 2) + rows.append(row) + +facts = {"script": S["fileName"].getValue() or None, "frame": rounded(S.context().getFrame()), + "fps": rounded(S["framesPerSecond"].getValue()), + "frame_range": [S["frameRange"]["start"].getValue(), S["frameRange"]["end"].getValue()]} +if FRAME != S.context().getFrame(): + facts["measured_at"] = rounded(FRAME) + +roots = [S if r in ("", "/") else node_of(r) for r in (P.get("roots") or ["/"])] +error_rows = [] +if P.get("errors", True): + STEP(step="errors", roots=["/" if r.isSame(S) else r.relativeName(S) for r in roots]) + with at_frame(FRAME): + error_rows, facts["error_scan"] = scan_errors() + if len(error_rows) >= ERRORS_MAX: + facts["error_scan"]["listed"] = ERRORS_MAX + + +# --- render: a low-resolution Cycles image from a separate gaffer python ------------------------------------- +RENDER_SCRIPT = r''' +import base64, os, time +import Gaffer, GafferImage, GafferScene, IECore, IECoreScene, imath +o = OPTIONS +facts = {"renderer": "Cycles"} +out = {"facts": facts} +try: + import GafferCycles +except ImportError as ex: + raise RuntimeError("render: Cycles is not available in this Gaffer (%s)" % ex) +def find(path): + found = script.descendant(str(path).strip().strip("/").replace("/", ".")) + if found is None: + raise ValueError("render.scene: no node or plug %r" % path) + return found +def scene_plug(target): + if isinstance(target, GafferScene.ScenePlug): + return target + for name in ("in", "out") if isinstance(target, GafferScene.Render) else ("out",): + if name in target and isinstance(target[name], GafferScene.ScenePlug): + return target[name] + raise ValueError("render.scene: %s has no scene output" % target.relativeName(script)) +def nodes(root): + for n in root.children(Gaffer.Node): + yield n + yield from nodes(n) +if o.get("scene"): + scene = scene_plug(find(o["scene"])) + facts["source"] = scene.relativeName(script) +else: + renders = [n for n in nodes(script) if isinstance(n, GafferScene.Render) and n["in"].getInput() is not None] + if renders: + scene = renders[0]["in"] + facts["source"] = "the scene of %s" % renders[0].relativeName(script) + else: + ends = [n["out"] for n in script.children(Gaffer.Node) if "out" in n and isinstance(n["out"], GafferScene.ScenePlug) and not n["out"].outputs()] + if len(ends) != 1: + raise ValueError("render: no Render node, and %d scene outputs lead nowhere (%s): give render.scene" + % (len(ends), ", ".join(e.relativeName(script) for e in ends[:8]))) + scene = ends[0] + facts["source"] = scene.relativeName(script) +frame = o["frame"] +context = Gaffer.Context(script.context()) +context.setFrame(frame) +with context: + g = scene.globals() + asked = g["option:render:resolution"].value if "option:render:resolution" in g else imath.V2i(4, 3) + k = o["size"] / float(max(asked.x, asked.y)) + w, h = max(16, int(round(asked.x * k))), max(16, int(round(asked.y * k))) + camera = o.get("camera") or (g["option:render:camera"].value if "option:render:camera" in g else "") + if not camera: + cams = sorted(scene.set("__cameras").value.paths()) + camera = cams[0] if cams else "" + if camera and not scene.exists(camera): + raise ValueError("render.camera: no location %s in the scene" % camera) +facts["camera"] = camera or "none in the scene: Gaffer's default camera, at the origin looking down -Z" +facts["resolution"] = [w, h] +script["opencodeVerifyOptions"] = GafferScene.StandardOptions() +opts = script["opencodeVerifyOptions"] +opts["in"].setInput(scene) +for name, value in (("render:resolution", imath.V2i(w, h)), ("render:resolutionMultiplier", 1.0)) + ((("render:camera", camera),) if camera else ()): + opts["options"][name]["enabled"].setValue(True) + opts["options"][name]["value"].setValue(value) +script["opencodeVerifyNoOutputs"] = GafferScene.DeleteOutputs() +script["opencodeVerifyNoOutputs"]["in"].setInput(opts["out"]) +script["opencodeVerifyNoOutputs"]["names"].setValue("*") +script["opencodeVerifyOutputs"] = GafferScene.Outputs() +outputs = script["opencodeVerifyOutputs"] +outputs["in"].setInput(script["opencodeVerifyNoOutputs"]["out"]) +exr = o["exr"] +outputs.addOutput("opencodeVerify", IECoreScene.Output(exr, "exr", "rgba", {})) +script["opencodeVerifyCycles"] = GafferCycles.CyclesOptions() +cycles = script["opencodeVerifyCycles"] +cycles["in"].setInput(outputs["out"]) +cycles["options"]["cycles:session:samples"]["enabled"].setValue(True) +cycles["options"]["cycles:session:samples"]["value"].setValue(o["samples"]) +script["opencodeVerifyRender"] = GafferScene.Render() +render = script["opencodeVerifyRender"] +render["in"].setInput(cycles["out"]) +render["renderer"].setValue("Cycles") +started = time.time() +with context: + render["task"].execute() +facts["seconds"] = round(time.time() - started, 2) +if not os.path.isfile(exr): + raise RuntimeError("the Cycles render wrote no image") +script["opencodeVerifyRead"] = GafferImage.ImageReader() +reader = script["opencodeVerifyRead"] +reader["fileName"].setValue(exr) +image = GafferImage.ImageAlgo.image(reader["out"]) +names = image.keys() +r, gr, b = (image[c] if c in names else None for c in ("R", "G", "B")) +alpha = image["A"] if "A" in names else None +n = len(r) +lum = [0.2126 * r[i] + 0.7152 * gr[i] + 0.0722 * b[i] for i in range(n)] +mean = sum(lum) / n +std = (sum((x - mean) ** 2 for x in lum) / n) ** 0.5 +facts["image"] = {"size": [w, h], "mean_luminance": round(mean, 4), "max_luminance": round(max(lum), 4), + "luminance_std": round(std, 4), "mean_alpha": round(sum(alpha) / n, 4) if alpha is not None else None, + "transparent_fraction": round(sum(1 for a in alpha if a < 0.01) / n, 4) if alpha is not None else None, + "black": max(lum) < 0.02, "uniform": std < 0.002, "values": "linear (scene-referred)"} +facts["frame"] = frame +script["opencodeVerifyWrite"] = GafferImage.ImageWriter() +writer = script["opencodeVerifyWrite"] +writer["in"].setInput(reader["out"]) +writer["fileName"].setValue(o["png"]) +with context: + writer["task"].execute() +with open(o["png"], "rb") as fh: + out["image_base64"] = base64.b64encode(fh.read()).decode("ascii") +''' + + +def render_options(spec): + spec = {} if spec is True else dict(spec) + unknown = set(spec) - {"scene", "camera", "size", "frame", "samples"} + if unknown: + raise ValueError("render: unknown keys %s; give {scene?, camera?, size?, frame?, samples?}" % sorted(unknown)) + size = int(spec.get("size", 320)) + if not 64 <= size <= 1024: + raise ValueError("render.size: 64 to 1024 pixels (longest side)") + samples = int(spec.get("samples", 16)) + if not 1 <= samples <= 1024: + raise ValueError("render.samples: 1 to 1024") + return {"scene": spec.get("scene"), "camera": spec.get("camera"), "size": size, "samples": samples, + "frame": float(spec["frame"]) if spec.get("frame") is not None else FRAME} + + +def launcher(job): + """`gaffer python `, started with this Gaffer's own launcher and environment, without a console window.""" + exe = os.path.join(os.environ.get("GAFFER_ROOT") or "", "bin", "gaffer.cmd" if os.name == "nt" else "gaffer") + if not os.path.isfile(exe): + raise RuntimeError("no Gaffer launcher at %s" % exe) + if os.name == "nt": + return '"%s" /d /s /c ""%s" python "%s""' % (os.environ.get("ComSpec") or "cmd.exe", exe, job) + return [exe, "python", job] + + +def child_env(role): + env = {k: v for k, v in os.environ.items() if k != "OPENCODE_GAFFER_OFFER_KEY"} + env["OPENCODE_GAFFER_ROLE"] = role # never offered to OpenCode + return env + + +def render_image(options): + work = tempfile.mkdtemp(prefix="opencode_verify_render_") + options["png"] = os.path.join(work, "verify.png").replace("\\", "/") + options["exr"] = os.path.join(work, "verify.exr").replace("\\", "/") + + def clean(): + for name in os.listdir(work): + try: + os.remove(os.path.join(work, name)) + except OSError: + pass + try: + os.rmdir(work) + except OSError: + pass + + if THROWAWAY: + # Already a background gaffer python on its own copy (gaffer_verify saved): render here. + scope = {"OPTIONS": options, "script": S, "__name__": "__opencode_render__"} + try: + exec(compile(RENDER_SCRIPT, "", "exec"), scope) # noqa: S102 + got = scope["out"] + finally: + clean() + got["facts"]["by"] = "this background gaffer python (on the file read)" + return got + started = time.time() + copy = os.path.join(work, os.path.basename(S["fileName"].getValue()) or "untitled.gfr").replace("\\", "/") + S.serialiseToFile(copy) + result_path = os.path.join(work, "result.json") + job = os.path.join(work, "render.py") + with open(job, "w", encoding="utf-8") as fh: + fh.write("import json, os, sys, traceback\nimport Gaffer\n") + fh.write("OPTIONS = json.loads(%r)\n" % json.dumps(options)) + fh.write("try:\n script = Gaffer.ScriptNode()\n script['fileName'].setValue(%r)\n" % copy) + fh.write(" script.load(continueOnError=True)\n") + fh.write(" exec(compile(%r, '', 'exec'))\n" % RENDER_SCRIPT) + fh.write("except Exception:\n out = {'error': traceback.format_exc()[-2000:]}\n") + fh.write("with open(%r, 'w', encoding='utf-8') as fh:\n json.dump(out, fh)\n" % result_path) + fh.write("sys.stdout.flush()\nos._exit(0)\n") + kwargs = {"stdin": subprocess.DEVNULL, "stdout": subprocess.PIPE, "stderr": subprocess.STDOUT, "env": child_env("verify"), "cwd": work} + if os.name == "nt": + kwargs["creationflags"] = 0x08000000 # no console window + try: + run = subprocess.run(launcher(job), timeout=int(P.get("timeout_s") or 300), **kwargs) + if not os.path.isfile(result_path): + tail = run.stdout.decode("utf-8", "replace")[-1500:] if run.stdout else "" + raise RuntimeError("the render gaffer python gave no result (exit %s):\n%s" % (run.returncode, tail)) + with open(result_path, encoding="utf-8") as fh: + got = json.load(fh) + finally: + clean() + if "error" in got: + raise RuntimeError("render: " + got["error"]) + got["facts"]["by"] = "a background gaffer python on a working copy of the open script (the open file keeps its name)" + got["facts"]["total_seconds"] = round(time.time() - started, 1) + return got + + +image = None +if P.get("render"): + STEP(step="render") + rendered = render_image(render_options(P["render"])) + facts["render"] = rendered["facts"] + image = rendered.get("image_base64") +facts["seconds"] = round(time.time() - STARTED, 2) +passed = sum(1 for r in rows if r["passed"]) +result = {"gaffer": {"version": Gaffer.About.versionString()}, + "passed": passed, "failed": len(rows) - passed, "all_passed": passed == len(rows), + "failed_labels": [r["label"] for r in rows if not r["passed"]], "criteria": rows, "errors": error_rows, + "facts": facts, **({"image_base64": image} if image else {})} diff --git a/integrations/gaffer/plugin/programs/verify_saved.py b/integrations/gaffer/plugin/programs/verify_saved.py new file mode 100644 index 0000000..9f50761 --- /dev/null +++ b/integrations/gaffer/plugin/programs/verify_saved.py @@ -0,0 +1,88 @@ +# SPDX-License-Identifier: MIT +# gaffer_verify with saved (fixed program): the same verification, read from the file as written on disk instead +# of the open script. This program only starts it and returns at once: a background `gaffer python`, started from +# this Gaffer (its launcher and environment: startup paths, extensions, variables), loads the .gfr into a fresh +# ScriptNode and runs the verify program there (criteria, errors, render). The plugin follows it from its log +# (OPENCODE_STEP lines: what it does now), bounds it with timeout_s and reads its result; Gaffer's main thread is +# never held while it runs. That process never offers itself to OpenCode (OPENCODE_GAFFER_ROLE=verify, no offer +# key) and opens no window. +# PARAMS: the verify parameters with "saved": true (the open file as saved) or a .gfr path; "timeout_s"?. +# JOB: verify.py, prepended by the plugin. +import json +import os +import subprocess +import tempfile +import time + +P = PARAMS # noqa: F821 +S = script # noqa: F821 +target = P.get("saved") +current = S["fileName"].getValue() +never_saved = not current or not os.path.isfile(current) +if target is True: + if never_saved: + raise ValueError("saved: the open script was never saved, there is no file to read (gaffer_save first)") + path = current +elif isinstance(target, str) and target: + base = os.path.dirname(current) if not never_saved else os.getcwd() + path = os.path.normpath(target if os.path.isabs(target) else os.path.join(base, target)) +else: + raise ValueError("saved: true (the open script as saved) or a .gfr path") +if not os.path.isfile(path): + raise ValueError("saved: no file %s" % path) + +exe = os.path.join(os.environ.get("GAFFER_ROOT") or "", "bin", "gaffer.cmd" if os.name == "nt" else "gaffer") +if not os.path.isfile(exe): + raise RuntimeError("no Gaffer launcher at %s" % exe) +params = {key: value for key, value in P.items() if key != "saved"} +work = tempfile.mkdtemp(prefix="opencode_verify_saved_") +out_path = os.path.join(work, "result.json") +log_path = os.path.join(work, "verify.log") +job = os.path.join(work, "verify.py") +with open(job, "w", encoding="utf-8") as fh: + fh.write( + "import json, os, sys, time, traceback\n" + "T0 = time.time()\n" + "def STEP(**what):\n" + " print('OPENCODE_STEP ' + json.dumps(dict(what, at_s=round(time.time() - T0, 1)), default=repr), flush=True)\n" + "STEP(step='load', file=%r)\n" + "import Gaffer, GafferScene, GafferImage, GafferDispatch, IECore, imath\n" + "warnings = []\n" + "try:\n" + " script = Gaffer.ScriptNode()\n" + " script['fileName'].setValue(%r)\n" + " with IECore.CapturingMessageHandler() as captured:\n" + " try:\n script.load(continueOnError=True)\n" + " except Exception as ex:\n warnings.append(str(ex)[:1500])\n" + " warnings += [('%%s: %%s' %% (m.context, m.message))[:1500] for m in captured.messages\n" + " if m.level <= IECore.Msg.Level.Warning][:20]\n" + " LOAD_S = round(time.time() - T0, 1)\n" + " STEP(step='loaded', seconds=LOAD_S)\n" + % (path, path.replace("\\", "/")) + ) + fh.write(" scope = {'Gaffer': Gaffer, 'GafferScene': GafferScene, 'GafferImage': GafferImage, 'GafferDispatch': GafferDispatch,\n" + " 'IECore': IECore, 'imath': imath, 'script': script, 'application': None, '__name__': '__opencode__',\n" + " 'result': {}, 'STEP': STEP, 'PARAMS': json.loads(%r)}\n" % json.dumps(params)) + fh.write(" exec(compile(%r, '', 'exec'), scope)\n" % JOB) # noqa: F821 - verify.py, from the plugin + fh.write(" result = scope['result']\n result['load_warnings'] = warnings\n result['load_seconds'] = LOAD_S\n") + fh.write("except Exception:\n result = {'error': traceback.format_exc()[-3000:]}\n") + # Written whole, then renamed: the plugin never reads half a result. + fh.write("with open(%r, 'w', encoding='utf-8') as _out:\n json.dump(result, _out, default=repr)\n" % (out_path + ".part")) + fh.write("os.replace(%r, %r)\nSTEP(step='done')\n" % (out_path + ".part", out_path)) + # Leave at once: the result is written, nothing in the exit handlers can lose it. + fh.write("sys.stdout.flush()\nos._exit(0)\n") +env = {k: v for k, v in os.environ.items() if k != "OPENCODE_GAFFER_OFFER_KEY"} +env["OPENCODE_GAFFER_ROLE"] = "verify" # never offered to OpenCode +env["PYTHONUNBUFFERED"] = "1" +with open(log_path, "wb") as log: + kwargs = {"stdin": subprocess.DEVNULL, "stdout": log, "stderr": subprocess.STDOUT, "env": env, "cwd": work} + if os.name == "nt": + kwargs["creationflags"] = 0x08000000 # no console window + command = '"%s" /d /s /c ""%s" python "%s""' % (os.environ.get("ComSpec") or "cmd.exe", exe, job) + else: + command = [exe, "python", job] + child = subprocess.Popen(command, **kwargs) +stat = os.stat(path) +result = {"started": {"pid": child.pid, "work": work, "log": log_path, "result": out_path, "file": path, + "saved_at": time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(stat.st_mtime)), + "bytes": stat.st_size, "timeout_s": int(P.get("timeout_s") or 540)}} diff --git a/integrations/gaffer/plugin/scripts/build-host.ts b/integrations/gaffer/plugin/scripts/build-host.ts index fa34a61..074882f 100644 --- a/integrations/gaffer/plugin/scripts/build-host.ts +++ b/integrations/gaffer/plugin/scripts/build-host.ts @@ -1,61 +1,68 @@ import { createHash } from "node:crypto" import { existsSync, mkdirSync, readdirSync, readFileSync, writeFileSync } from "node:fs" -import { dirname, join, relative, sep } from "node:path" +import { dirname, join, relative } from "node:path" import { fileURLToPath } from "node:url" import { build } from "esbuild" -import { writeStoredZip } from "../src/zip.ts" +import { writeStoredZip } from "../../../../packages/integration-kit/src/zip.ts" const root = fileURLToPath(new URL("..", import.meta.url)) -const hostRoot = join(root, "..", "host") +const host = join(root, "..", "host") const pkgPath = join(root, "package.json") const pkg = JSON.parse(readFileSync(pkgPath, "utf8")) -const manifest = JSON.parse(readFileSync(join(hostRoot, "manifest.json"), "utf8")) -if (manifest.version !== pkg.version) throw new Error("Host and plugin versions must match.") +const manifest = JSON.parse(readFileSync(join(host, "manifest.json"), "utf8")) +if (manifest.version !== pkg.version) throw new Error("Host and plugin versions differ") const check = process.argv.includes("--check") -const sidecar = await build({ +const bundle = await build({ entryPoints: [join(root, "src", "sidecar.ts")], bundle: true, - write: false, - format: "esm", + format: "cjs", platform: "node", target: "node22", - packages: "bundle", - minify: true, - banner: { - js: 'import { createRequire as __createRequire } from "node:module"; const require = __createRequire(import.meta.url);', - }, + write: false, + legalComments: "eof", + logLevel: "warning", }) -function walk(dir: string): string[] { - return readdirSync(dir, { withFileTypes: true }) - .flatMap((e) => (e.isDirectory() ? (e.name === "__pycache__" ? [] : walk(join(dir, e.name))) : [join(dir, e.name)])) - .sort() -} -const entries = walk(hostRoot) - .filter((f) => !f.endsWith(".pyc")) - .map((file) => ({ - name: relative(hostRoot, file).split(sep).join("/"), - data: Buffer.from(readFileSync(file, "utf8").replaceAll("\r\n", "\n")), - })) -const sidecarFile = sidecar.outputFiles?.[0] -if (!sidecarFile) throw new Error("Sidecar build produced no JavaScript.") -entries.push({ name: "sidecar.js", data: Buffer.from(sidecarFile.contents) }) -const bytes = writeStoredZip(entries.sort((a, b) => a.name.localeCompare(b.name, "en"))) -const payload = `host/gaffer/opencode_link-${pkg.version}.zip` -const sha256 = createHash("sha256").update(bytes).digest("hex") -const file = join(root, payload) +const sidecar = Buffer.from(bundle.outputFiles?.[0]?.contents ?? []) +const walk = (dir: string): string[] => + readdirSync(dir, { withFileTypes: true }).flatMap((entry) => + entry.name === "__pycache__" || entry.name.endsWith(".pyc") || entry.name === "sidecar.cjs" + ? [] + : entry.isDirectory() + ? walk(join(dir, entry.name)) + : [join(dir, entry.name)], + ) +const entries = walk(host).map((file) => ({ + name: relative(host, file).replaceAll("\\", "/"), + data: Buffer.from(readFileSync(file, "utf8").replaceAll("\r\n", "\n")), +})) +entries.push({ name: "sidecar.cjs", data: sidecar }) +// The kit's discovery client, which the installed part's advertiser uses to offer Gaffer's scripts to OpenCode. +entries.push({ + name: "python/GafferOpenCode/opencode_service.py", + data: Buffer.from( + readFileSync( + join(root, "..", "..", "..", "packages", "integration-kit", "python", "opencode_service.py"), + "utf8", + ).replaceAll("\r\n", "\n"), + ), +}) +const payload = writeStoredZip(entries.sort((a, b) => (a.name < b.name ? -1 : 1))) +const name = `host/gaffer/opencode_link-${pkg.version}.zip` +const digest = createHash("sha256").update(payload).digest("hex") +const file = join(root, name) if (check) { if ( - pkg.opencodeIntegration.host.payload !== payload || - pkg.opencodeIntegration.host.sha256 !== sha256 || !existsSync(file) || - !readFileSync(file).equals(bytes) + !readFileSync(file).equals(payload) || + pkg.opencodeIntegration.host.sha256 !== digest || + pkg.opencodeIntegration.host.payload !== name ) - throw new Error("Host payload is stale; run build:host.") + throw new Error("Gaffer host payload is stale: run npm run build:host --workspace @neuralnomads/opencode-gaffer") } else { mkdirSync(dirname(file), { recursive: true }) - writeFileSync(file, bytes) - pkg.opencodeIntegration.host.payload = payload - pkg.opencodeIntegration.host.sha256 = sha256 + writeFileSync(file, payload) + pkg.opencodeIntegration.host.payload = name + pkg.opencodeIntegration.host.sha256 = digest writeFileSync(pkgPath, `${JSON.stringify(pkg, null, 2)}\n`) } -console.log(`gaffer host: ${check ? "verified" : "built"}`) +console.log(`Gaffer host ${check ? "verified" : "built"}: ${pkg.version}`) diff --git a/integrations/gaffer/plugin/scripts/build.ts b/integrations/gaffer/plugin/scripts/build.ts index f8f0076..2fcecf2 100644 --- a/integrations/gaffer/plugin/scripts/build.ts +++ b/integrations/gaffer/plugin/scripts/build.ts @@ -1,28 +1,38 @@ +// Builds dist/: one ESM file bundling the plugin and the integration kit (the SDK stays external, +// the OpenCode binary provides it), and its type declaration. Fails when the host artifacts are stale. +// Usage: node scripts/build.ts + import { execFileSync } from "node:child_process" -import { mkdirSync, writeFileSync } from "node:fs" +import { mkdirSync, rmSync, writeFileSync } from "node:fs" import { join } from "node:path" import { fileURLToPath } from "node:url" + import { build } from "esbuild" const root = fileURLToPath(new URL("..", import.meta.url)) -execFileSync(process.execPath, [join(root, "scripts", "build-host.ts"), "--check"], { - stdio: "inherit", - windowsHide: true, -}) -mkdirSync(join(root, "dist"), { recursive: true }) +const dist = join(root, "dist") + +execFileSync(process.execPath, [join(root, "scripts", "build-host.ts"), "--check"], { stdio: "inherit" }) + +rmSync(dist, { recursive: true, force: true }) +mkdirSync(dist, { recursive: true }) await build({ entryPoints: [join(root, "src", "plugin.ts")], - outfile: join(root, "dist", "index.js"), + outfile: join(dist, "index.js"), bundle: true, format: "esm", platform: "node", target: "node22", external: ["@opencode/plugin", "@opencode/plugin/*"], + // semver is CommonJS: give the bundle a require for Node's built-in modules. banner: { - js: 'import { createRequire as __createRequire } from "node:module"; const require = __createRequire(import.meta.url);', + js: 'import { createRequire as __createRequire } from "node:module";\nconst require = __createRequire(import.meta.url);', }, + legalComments: "eof", + logLevel: "warning", }) writeFileSync( - join(root, "dist", "index.d.ts"), - 'import type { Plugin } from "@opencode/plugin"\ndeclare const plugin: Plugin.Plugin\nexport default plugin\n', + join(dist, "index.d.ts"), + 'import type { Plugin } from "@opencode/plugin"\n\ndeclare const plugin: Plugin.Plugin\nexport default plugin\n', ) +console.log("build: dist/index.js") diff --git a/integrations/gaffer/plugin/skills/gaffer-graph/SKILL.md b/integrations/gaffer/plugin/skills/gaffer-graph/SKILL.md deleted file mode 100644 index 24b7065..0000000 --- a/integrations/gaffer/plugin/skills/gaffer-graph/SKILL.md +++ /dev/null @@ -1,38 +0,0 @@ ---- - -name: gaffer-graph -description: Inspect and edit the Gaffer graph explicitly attached to the current OpenCode session. ---- - -# Gaffer - -Use this skill only when a Gaffer script is attached to the current session. - -## Workflow - -1. Read the binding metadata in the system context and pass its exact `script_handle` to every Gaffer tool call. -2. Call `gaffer_context` with a small working set. Prefer explicit refs, a narrow match, `plugs` projections and the smallest useful radius. Do not request complete node schemas when a few plug globs are sufficient. -3. Preserve the returned `revision`. Do not prepare a write from stale context. -4. Call `gaffer_apply` with that exact `if_revision`, a new stable `command_id`, and one coherent atomic intent. -5. Include postconditions in `expect` for the objects and values that determine success. -6. On `revision_conflict`, read context again and rebuild the intent instead of retrying it unchanged. - -## Operations - -`gaffer_apply` supports graph creation and editing plus complete batch rendering. `graph.connect_many` connects ordered `sources` to an input `ArrayPlug` `target`, growing it safely. `output.define` calls the native `GafferScene.Outputs.addOutput()` with `target`, `name`, `file_name`, `driver`, `channels`, and optional scalar `parameters`. - -Launch a configured `GafferScene.Render` with a separate intent containing only `{ "op": "render.start", "target": "...", "name": "..." }`. Do not add `expect` to render intents. The result returns a job ID immediately while `LocalDispatcher` renders out of process. Poll with `gaffer_context` and `job: { "id": "job:...", "after": 0 }` until `state` is terminal. A successful result includes output existence, dimensions, channels, hash, min/max/average statistics, and a PNG preview attached directly for visual feedback. Cancel with a separate `render.cancel` intent whose `target` is the job ID. - -Scalar, numeric-vector and string-vector plug values are supported. `ui.view` focuses a node and may select a registered SceneView renderer with `renderer`. For an interactive GPU-only Cycles viewport, use `{ "op": "ui.view", "target": "...", "renderer": "Cycles", "parameters": { "device": "OPTIX:*" } }`. The host verifies that an OptiX device exists, sets `renderer.cycles.device` before enabling Cycles, and does not include CPU. - -Use the exact operation fields from the tool schema: `target` for `plug.set_value`, `node.rename`, `graph.delete`, and `ui.view`; `source` plus `destination` for `graph.connect`. Postconditions use `{ "subject": "...", "predicate": "exists|name_equals|value_equals|connected_to", "value": ... }`. Do not substitute `plug`, `node`, `ref`, or `kind` for these fields. - -Use opaque refs returned by `gaffer_context`. A `node.create` operation may set `bind` to a simple identifier, and later operations in the same intent may reference the new node as `$identifier` or a descendant such as `$identifier.plugName`. Returned opaque refs also support descendants, for example `node:opaque.options.render:resolution.value`. - -For simple nodes, `node.create` accepts a canonical `Namespace::Type`. For parameterised or plugin nodes, query `gaffer_context` with `recipe_match` and use the returned recipe `id` in the operation's `recipe` field. These recipes are discovered from the live Gaffer `NodeMenu`, so renderer plugins and shader/light variants match the current installation. - -Request `include: ["capabilities"]` only when operation or node-type discovery is needed. Request `include: ["recipes"]` or supply `recipe_match` only for native creation discovery; ordinary graph reads omit the full catalogues to keep responses compact. - -Gaffer may sanitise a requested node name or append a numeric suffix on collision. Read the effective `name`, `path`, and `ref` from the `created` result instead of assuming the requested name was retained. - -Do not invent refs or switch scripts implicitly. diff --git a/integrations/gaffer/plugin/src/closed.ts b/integrations/gaffer/plugin/src/closed.ts new file mode 100644 index 0000000..7c19d79 --- /dev/null +++ b/integrations/gaffer/plugin/src/closed.ts @@ -0,0 +1,11 @@ +// What became of a Gaffer that stopped answering a conversation (the kit's crashWatch): closed, or alive without +// renewing its offer (frozen). Gaffer writes no crash report: a crash reads as a closed process. + +import { crashWatch } from "./kit.ts" + +export const { crashReport, closedMessage, silentMessage } = crashWatch({ + appName: "Gaffer", + app: "gaffer", + part: "OpenCode Link", + isReport: () => false, +}) diff --git a/integrations/gaffer/plugin/src/host-client.ts b/integrations/gaffer/plugin/src/host-client.ts deleted file mode 100644 index e48b830..0000000 --- a/integrations/gaffer/plugin/src/host-client.ts +++ /dev/null @@ -1,13 +0,0 @@ -import { requestHost as request } from "./kit.ts" -export type HostBinding = { host: string; port: number; token: string; scriptHandle: string } -export async function requestHost( - binding: HostBinding, - method: string, - params: Record = {}, - options: { signal?: AbortSignal; timeoutMs?: number } = {}, -): Promise> { - return (await request(binding, method, { ...params, script_handle: binding.scriptHandle }, options)) as Record< - string, - unknown - > -} diff --git a/integrations/gaffer/plugin/src/host-install.ts b/integrations/gaffer/plugin/src/host-install.ts index 2eebeb1..2511598 100644 --- a/integrations/gaffer/plugin/src/host-install.ts +++ b/integrations/gaffer/plugin/src/host-install.ts @@ -1,45 +1,81 @@ -import { createHash } from "node:crypto" -import { existsSync, mkdirSync, readFileSync, renameSync, writeFileSync } from "node:fs" -import { homedir } from "node:os" -import { dirname, isAbsolute, join } from "node:path" -import type { InstallHostOutput, IntegrationMarker } from "./kit.ts" -import { packageRoot, stateRoot } from "./paths.ts" -import { readStoredZip } from "./zip.ts" +// The installed part: a content-addressed payload in the plugin's state folder, started by a managed loader in +// Gaffer's GUI startup folder (neuralnomadsOpenCode.py). Gaffer runs every startup file of that folder when its +// interface starts. -const HEADER = "# Managed by neuralnomads.gaffer\n" -export async function installHost( - marker: IntegrationMarker, - options: Record = {}, -): Promise { - const startup = - options.hostStartupDir ?? - process.env.NEURALNOMADS_GAFFER_HOST_STARTUP_DIR ?? - join(homedir(), "gaffer", "startup", "gui") - if (typeof startup !== "string" || !isAbsolute(startup)) - return { outcome: "refused", detail: "hostStartupDir must be an absolute Gaffer GUI startup directory." } - const loader = join(startup, "neuralnomadsOpenCode.py") +import { createHash, randomUUID } from "node:crypto" +import { existsSync, mkdirSync, readFileSync, renameSync, rmSync, statSync, writeFileSync } from "node:fs" +import { dirname, isAbsolute, join, resolve } from "node:path" +import { readStoredZip } from "../../../../packages/integration-kit/src/zip.ts" +import type { HostInspection, InstallHostOutput, IntegrationMarker } from "./kit.ts" +import { packageRoot, stateDir } from "./paths.ts" + +export const LOADER = "neuralnomadsOpenCode.py" +export const HEADER = "# Managed by neuralnomads.gaffer\n" + +// The installed version: the manifest of the payload the managed loader starts. +export function inspectHost(startup?: string): HostInspection { + if (!startup || !existsSync(startup)) return { kind: "not-inspected" } + const loader = join(startup, LOADER) + try { + if (statSync(loader).size > 64 * 1024) return { kind: "not-inspected" } + const source = readFileSync(loader, "utf8") + if (!source.startsWith(HEADER)) return { kind: "not-inspected" } + const started = /run_path\("([^"]+)\/startup\/gui\/openCode\.py"/.exec(source)?.[1] + if (!started) return { kind: "not-inspected" } + const version = JSON.parse(readFileSync(join(started, "manifest.json"), "utf8")).version + return typeof version === "string" ? { kind: "found", version } : { kind: "not-inspected" } + } catch (error) { + return (error as NodeJS.ErrnoException).code === "ENOENT" ? { kind: "not-found" } : { kind: "not-inspected" } + } +} + +export function installHost(marker: IntegrationMarker, startup?: string, state = stateDir()): InstallHostOutput { + if (!startup || !isAbsolute(startup)) + return { outcome: "refused", detail: "hostStartupDir must be an absolute Gaffer GUI startup folder." } + const loader = join(startup, LOADER) if (existsSync(loader) && !readFileSync(loader, "utf8").startsWith(HEADER)) - return { outcome: "refused", detail: "The existing startup file is not managed by this integration." } + return { outcome: "refused", detail: `${loader} exists and is not managed by this integration.` } const host = marker.host if (!host) return { outcome: "refused" } - const bytes = readFileSync(join(packageRoot, host.payload)) - if (createHash("sha256").update(bytes).digest("hex") !== host.sha256) + const archive = readFileSync(join(packageRoot, host.payload)) + if (createHash("sha256").update(archive).digest("hex") !== host.sha256) return { outcome: "failed", detail: "Host payload checksum mismatch." } - const entries = readStoredZip(bytes) - const target = join(stateRoot(), "hosts", host.sha256) - for (const { name, data } of entries) { - const file = join(target, name) - mkdirSync(dirname(file), { recursive: true }) - writeFileSync(file, data) + const entries = readStoredZip(archive) + const payload = join(state, "hosts", host.sha256) + const staging = `${payload}-${randomUUID()}` + const source = `${HEADER}import runpy\nrunpy.run_path(${JSON.stringify( + `${resolve(payload).replaceAll("\\", "/")}/startup/gui/openCode.py`, + )}, init_globals={"application": application})\n` + const current = inspectHost(startup) + const same = + existsSync(payload) && + entries.every((entry) => { + const file = join(payload, entry.name) + return existsSync(file) && readFileSync(file).equals(entry.data) + }) + if (same && existsSync(loader) && readFileSync(loader, "utf8") === source) return { outcome: "unchanged" } + // Content-addressed immutable payload: a damaged directory is never silently trusted or overwritten. + if (existsSync(payload) && !same) + return { outcome: "failed", detail: "The installed host payload is damaged; remove its folder before retrying." } + const temporary = `${loader}-${randomUUID()}` + try { + if (!same) { + for (const entry of entries) { + const file = join(staging, entry.name) + mkdirSync(dirname(file), { recursive: true }) + writeFileSync(file, entry.data) + } + renameSync(staging, payload) + } + mkdirSync(startup, { recursive: true }) + writeFileSync(temporary, source) + renameSync(temporary, loader) + } finally { + rmSync(staging, { recursive: true, force: true }) + rmSync(temporary, { force: true }) + } + return { + outcome: current.kind === "found" ? "updated" : "installed", + detail: "Restart Gaffer: OpenCode Link offers its scripts to OpenCode when it starts.", } - const source = - HEADER + - `import runpy\nrunpy.run_path(${JSON.stringify(join(target, "startup", "gui", "openCode.py").replaceAll("\\", "/"))}, init_globals={"application": application})\n` - if (existsSync(loader) && readFileSync(loader, "utf8") === source) return { outcome: "unchanged" } - const updated = existsSync(loader) - mkdirSync(startup, { recursive: true }) - const temporary = `${loader}.${process.pid}.tmp` - writeFileSync(temporary, source) - renameSync(temporary, loader) - return { outcome: updated ? "updated" : "installed" } } diff --git a/integrations/gaffer/plugin/src/integration.ts b/integrations/gaffer/plugin/src/integration.ts index c3f8f9b..6d97a1f 100644 --- a/integrations/gaffer/plugin/src/integration.ts +++ b/integrations/gaffer/plugin/src/integration.ts @@ -1,37 +1,127 @@ +// OpenCode for Gaffer: RPC status contract, tools and the working method. Loading has no side effect: no socket, +// process or installation; the host is reached only once a conversation is attached to an offer validated by +// hello. The conversations' attachment and host calls are the kit's host session, as for Blender and Houdini. + import { readFileSync } from "node:fs" import { join } from "node:path" import { pathToFileURL } from "node:url" + import type { Plugin } from "@opencode/plugin" -import { installHost } from "./host-install.ts" +import type { ToolContext } from "@opencode/plugin/promise/tool" + +import { closedMessage, crashReport, silentMessage } from "./closed.ts" +import { inspectHost, installHost } from "./host-install.ts" import { + type AttachmentStorage, + buildPluginTools, + createAttachments, + createHostSession, createIntegrationRpc, HELLO_TIMEOUT_MS, + HostError, + hostErrorWith, type IntegrationMarker, + isWslDaemon, + loadMethodRegistry, type OfferRegistry, + readScript, + registerPluginTools, requestHost, + type SessionBindings, sharedOfferRegistry, + type ToolContent, + trackAdvertise, } from "./kit.ts" -import { packageRoot } from "./paths.ts" -import { applyInput, contextInput } from "./schemas.ts" -import { loadSkills } from "./skills.ts" +import { legacyInstalls } from "./legacy.ts" +import { type EntryPut, methodAttacher } from "./method.ts" +import { type GafferOptions, readOptions } from "./options.ts" +import { defaultPathEnv, defaultStartupDir, methodsDir, type PathEnv, packageRoot } from "./paths.ts" +import { type ProgramName, programCode } from "./programs.ts" +import { isSavedStart, waitSaved } from "./saved-verify.ts" +import { addTask, cancelTask, getTask, isRenderStart, taskStatus } from "./tasks.ts" + +export const APP = "gaffer" +const NAMESPACE_DESCRIPTION = "Gaffer, through OpenCode Link: read and change the attached script." + +export const PERMISSIONS = { + attach: "gaffer_attach", + install: "gaffer_install_host", + executeCode: "gaffer_execute_code", + write: "gaffer_write", +} as const + +const MAX_TEXT = 100_000 +// A verification may compute the graph, render, or start a background Gaffer on the saved file. +const VERIFY_TIMEOUT_MS = 600_000 +// Longest wait of one gaffer_render call that follows a task. +export const RENDER_WAIT_MAX_S = 3_600 + +const HINTS: Record = { + stale_handle: "The attached Gaffer script was closed: gaffer_hosts shows the open scripts.", + stale_revision: "The script changed since your last read: call gaffer_context and use its revision.", + expectation_failed: "An expect clause does not hold: nothing was changed.", + busy: "Gaffer is busy (a modal dialog or a long computation): try again shortly.", + quarantined: + "A failed change could not be proven undone: read the script (gaffer_context), check it, then give that read's revision as release_quarantine.", + method_disabled: "Code execution is not available in the attached Gaffer.", + method_not_found: + "The attached Gaffer's OpenCode Link lacks this method: install this plugin's OpenCode Link (gaffer_install_host) and restart Gaffer.", + invalid_args: "Gaffer refused the request.", + unreachable: "The attached Gaffer is unreachable: it may have closed or stopped offering its script.", + timeout: + "Gaffer did not answer in time; the change may still complete. Call gaffer_context, then retry with the same command_id if it did not: a replayed command_id applies nothing twice.", + cancelled: + "The request was cancelled; a change may still complete in Gaffer. Call gaffer_context, then retry with the same command_id if it did not.", +} + +// The installed part's own text and data follow the hint (the kit's message after its "Host refused" prefix). +const hostError = hostErrorWith(HINTS, "Gaffer", (error: HostError) => + error.message.replace(/^Host refused the request \([^)]*\)[.:]?\s*/, ""), +) -export function readPackage(): { version: string; opencodeIntegration: IntegrationMarker } { - return JSON.parse(readFileSync(join(packageRoot, "package.json"), "utf8")) +export type GafferDeps = { + env?: NodeJS.ProcessEnv + paths?: PathEnv + // A private registry for tests; the daemon shares one registry across Locations. + registry?: OfferRegistry + // The clock of the stored attachments' lease, for tests. + now?: () => number + wsl?: () => boolean + // The installed payload's folder, for tests. + state?: string +} + +type PackageJson = { version: string; opencodeIntegration: IntegrationMarker } + +export function readPackage(): PackageJson { + return JSON.parse(readFileSync(join(packageRoot, "package.json"), "utf8")) as PackageJson } // The offer list shared by the daemon's Locations, named after this version's host contract (the kit's -// sharedOfferRegistry): after an upgrade without restarting OpenCode, the new host is judged by this range. +// sharedOfferRegistry): after an upgrade without restarting OpenCode, the new host is judged by this range. One +// Gaffer offers each of its scripts: hello names the offer's script. export function createRegistry(marker: IntegrationMarker): OfferRegistry { const host = marker.host as NonNullable - return sharedOfferRegistry("gaffer", { + return sharedOfferRegistry(APP, { expected: { protocol: host.protocol, range: host.range, appVersions: host.appVersions }, hello: (endpoint, scriptHandle) => requestHost(endpoint, "hello", { script_handle: scriptHandle }, { timeoutMs: HELLO_TIMEOUT_MS }), }) } -export function resultContent(result: Record) { - const clean = JSON.parse(JSON.stringify(result)) +function bounded(text: string) { + return text.length > MAX_TEXT + ? `${text.slice(0, MAX_TEXT)}\n[truncated: ${text.length - MAX_TEXT} more characters]` + : text +} + +function json(value: unknown) { + return bounded(JSON.stringify(value)) +} + +// A render started by gaffer_apply render.start: its preview images are attachments, never paths in the text. +export function resultContent(result: Record): string | ToolContent[] { + const clean = JSON.parse(JSON.stringify(result ?? {})) const jobs = [clean.job, ...(clean.jobs ?? [])].filter(Boolean) as Array<{ outputs?: Array<{ preview_path?: string; name?: string }> }> @@ -49,134 +139,327 @@ export function resultContent(result: Record) { : [], ), ) - // Image paths are attachment URIs, never part of the text content. for (const job of jobs) for (const output of job.outputs ?? []) delete output.preview_path - const text = JSON.stringify(clean, null, 2) - return images.length ? [{ type: "text" as const, text }, ...images] : text + const text = json(clean) + return images.length ? [{ type: "text", text }, ...images] : text +} + +// After a save: what reads the file back, as written on disk (the open script can differ from it). +export function verifySavedHint(saved: true | string) { + return `gaffer_verify with saved: ${saved === true ? "true" : JSON.stringify(saved)} reads what this file holds.` } -export async function setupGaffer(ctx: Plugin.Context, deps: { registry?: OfferRegistry; wsl?: () => boolean } = {}) { +type ProgramRun = { result?: unknown; status?: string; message?: string; traceback?: string; hint?: string } + +export async function setupGaffer(ctx: Plugin.Context, deps: GafferDeps = {}) { + const env = deps.env ?? process.env + const paths = deps.paths ?? defaultPathEnv() + const options: GafferOptions = readOptions(ctx.options as Record | undefined, env) const pkg = readPackage() const marker = pkg.opencodeIntegration - if (!marker.host) throw new Error("Gaffer requires a native host marker.") const registry = deps.registry ?? createRegistry(marker) + const startup = () => options.hostStartupDir ?? defaultStartupDir(paths) + const legacy = () => legacyInstalls(paths, options.hostStartupDir) + const manifests = loadMethodRegistry(methodsDir) + const rpc = createIntegrationRpc({ marker, version: pkg.version, registry, - wsl: deps.wsl, nativeSession: { location: ctx.location, - get: ctx.session?.get ? (input) => ctx.session.get(input as never) : undefined, + get: ctx.session?.get ? (input) => ctx.session.get(input as Parameters[0]) : undefined, + }, + inspectHost: async () => inspectHost(startup()), + installHost: async () => installHost(marker, startup(), deps.state), + wsl: deps.wsl, + notices: async () => (legacy().length ? ["legacy_install"] : []), + }) + const bindings = rpc.bindings as SessionBindings + // Attachments kept across an OpenCode restart and a file change (a new offer under the same key). + const attachments = createAttachments({ + app: APP, + registry, + storage: (ctx as unknown as { storage?: AttachmentStorage }).storage, + now: deps.now, + owns: (sessionID) => bindings.snapshot(sessionID), + }) + const handlers: Record Promise> = { + ...rpc.handlers, + advertise: trackAdvertise(registry, (input) => rpc.handlers.advertise?.(input) as Promise, APP), + attachHost: async (input: unknown, context?: { signal?: AbortSignal }) => { + const result = (await rpc.handlers.attachHost?.(input, context)) as { outcome?: string } + const sessionID = (input as { sessionID?: unknown } | null)?.sessionID + if (result?.outcome === "attached" && typeof sessionID === "string") await attachments.remember(sessionID) + return result + }, + // The stored attachment goes first, once the session is verifiably this Location's: the detach's own change + // event must not find it and attach the conversation back. + detachHost: async (input: unknown, context?: { signal?: AbortSignal }) => { + const sessionID = (input as { sessionID?: unknown } | null)?.sessionID + if (typeof sessionID === "string" && (await bindings.snapshot(sessionID, context?.signal))) + await attachments.forget(sessionID) + return rpc.handlers.detachHost?.(input, context) }, - installHost: () => installHost(marker, ctx.options as Record | undefined), + } + + const assert = async ( + context: ToolContext, + action: string, + resources: string[], + metadata?: Record, + ) => { + await ctx.permission.assert({ + sessionID: context.sessionID, + agent: context.agent, + action, + resources, + ...(metadata ? { metadata } : {}), + source: { type: "tool", messageID: context.messageID, id: context.id }, + } as Parameters[0]) + } + + // The method joins a conversation when it uses a Gaffer script (an instructions entry, where OpenCode has them). + const instructions = (ctx as unknown as { session?: { instructions?: { entry?: { put?: EntryPut } } } }).session + const put = instructions?.instructions?.entry?.put + const attachMethod = methodAttacher(put ? (input) => put.call(instructions?.instructions?.entry, input) : undefined) + + const hostSession = createHostSession({ + texts: { + appName: "Gaffer", + app: APP, + part: "OpenCode Link", + partKind: "OpenCode Link", + noneOffered: + "No Gaffer script is offered: start Gaffer 1.6 or later with OpenCode Link installed (gaffer_install_host); it offers each open script when it starts.", + installHint: `install OpenCode Link ${pkg.version} (gaffer_install_host) and restart Gaffer`, + }, + registry, + bindings, + attachments, + attachMethod, + crash: { crashReport, closedMessage, silentMessage }, + wsl: deps.wsl ?? (() => isWslDaemon()), + requestTimeoutMs: options.requestTimeoutMs, + hostError, }) - const bindings = rpc.bindings - if (!bindings) throw new Error("Gaffer requires native session bindings.") - const registrations: Array<{ dispose(): Promise }> = [] + const { attachedTo, ticket, callHost, maintain } = hostSession + + // Every host call names its conversation: the installed part keeps per-conversation cancellation. + const call = ( + context: ToolContext, + method: string, + params: Record, + timeoutMs?: number, + captured?: Parameters[4], + ) => callHost(context, method, { session_id: context.sessionID, ...params }, timeoutMs, captured) + + // A fixed program through the host's code execution, read only (programs.ts). + const program = async (context: ToolContext, name: ProgramName, params: unknown, timeoutMs?: number) => { + const out = (await call( + context, + "execute_code", + { code: programCode(name, params), read_only: true }, + timeoutMs, + )) as ProgramRun + if (out.status === "failed") + throw new Error( + `gaffer_${name}: ${String(out.message ?? "")}${out.hint ? `\n${out.hint}` : ""}\n${String(out.traceback ?? "").slice(-1200)}`, + ) + return out.result + } + + const withImage = (value: unknown, name: string): string | ToolContent[] => { + const { image_base64, ...rest } = (value ?? {}) as { image_base64?: string } & Record + const text = json(rest) + if (typeof image_base64 !== "string" || !image_base64) return text + return [ + { type: "text", text }, + { type: "file", uri: `data:image/png;base64,${image_base64}`, mime: "image/png", name }, + ] + } + + const follow = async (id: string, waitS: unknown, context: ToolContext) => { + const task = getTask(id) + if (!task) throw new Error(`No render task ${id} in this OpenCode.`) + // A whole render can be waited for in one call (the tool call itself has no time limit). + const until = Date.now() + Math.min(Math.max(Number(waitS ?? 0), 0), RENDER_WAIT_MAX_S) * 1_000 + let status = taskStatus(task) + while (status.state === "rendering" && Date.now() < until && !context.signal.aborted) { + await new Promise((resolve) => setTimeout(resolve, 500)) + status = taskStatus(task) + } + return status + } + + const executors: Record< + string, + (args: Record, context: ToolContext) => Promise + > = { + async gaffer_hosts(args, context) { + const out = await hostSession.hosts(args, context, { + allow: (held, offer) => assert(held, PERMISSIONS.attach, [offer.label], { label: offer.label }), + status: () => rpc.status(), + list: "scripts", + }) + const found = legacy() + // Two Gaffers with the same file open offer the same label: said, so that the right one is attached. + const labels = registry.hosts().offers.map((offer) => offer.label) + const twice = [...new Set(labels.filter((label, index) => labels.indexOf(label) !== index))] + return json({ + ...out, + ...(twice.length + ? { + same_label: `Several Gaffers have the same file open (${twice.join(", ")}): their offers differ by offerID only; attach the one you mean, or close the others.`, + } + : {}), + ...(found.length + ? { + legacy_install: { + found, + fix: "An earlier OpenCode installation also loads in Gaffer (another editor and a package of the same name): remove them, then restart Gaffer.", + }, + } + : {}), + }) + }, + + async gaffer_install_host(_args, context) { + await assert(context, PERMISSIONS.install, ["*"]) + return json(await handlers.installHost?.({})) + }, + + async gaffer_context(args, context) { + const { release_quarantine: release, ...query } = args + if (release === undefined) + return json({ + ...((await call(context, "context", query)) as object), + attached_to: attachedTo(context.sessionID), + }) + // After a failed change whose undo could not be proven: the agent read the script (this revision) and lifts + // the quarantine; the script is read again with it. + const released = (await call(context, "release", { revision: release })) as { released?: boolean } + return json({ + ...((await call(context, "context", query)) as object), + attached_to: attachedTo(context.sessionID), + released: released.released === true, + }) + }, + + // The typed intent, unchanged: one undo step, with the write permission. + async gaffer_apply(args, context) { + const held = await ticket(context) + await assert(context, PERMISSIONS.write, [held.label ?? "*"], { operations: args.operations }) + return resultContent((await call(context, "apply", args, undefined, held)) as Record) + }, + + // The dangerous method: every run asks permission. + async gaffer_execute_code(args, context) { + // A script file is read here, byte for byte: no escaping of the code through a JSON or JavaScript string. + const script = + typeof args.path === "string" && args.path ? readScript(args.path, ctx.location?.directory) : undefined + if (script && args.code !== undefined) throw new Error("Give code or path, not both.") + const code = script?.code ?? String(args.code ?? "") + if (!code.trim()) throw new Error(script ? `${script.path} is empty.` : "code or path is required.") + const held = await ticket(context) + if (!held.bound.methods.includes("execute_code")) throw new HostError("method_disabled", "") + await assert(context, PERMISSIONS.executeCode, [script?.path ?? "*"], { + ...(script ? { path: script.path } : {}), + code: code.length > 4_000 ? `${code.slice(0, 4_000)}\n…` : code, + }) + const seconds = typeof args.timeout_s === "number" ? Math.min(Math.max(args.timeout_s, 1), 3_600) : undefined + const out = (await call( + context, + "execute_code", + { + code, + ...(script ? { filename: script.path } : {}), + ...(args.read_only === true ? { read_only: true } : {}), + }, + seconds ? seconds * 1_000 : undefined, + held, + )) as Record + return json(out.saved === true ? { ...out, check: verifySavedHint(true) } : out) + }, + + async gaffer_inspect(args, context) { + if (args.search === undefined && args.type === undefined && args.node === undefined && args.gaffer === undefined) + throw new Error("Give search, type, node and/or gaffer.") + return json(await program(context, "inspect", args)) + }, + + async gaffer_verify(args, context) { + // saved: the file as written on disk, read by a background Gaffer, instead of the open script. + const saved = args.saved !== undefined && args.saved !== false + if (saved && args.saved !== true && typeof args.saved !== "string") + throw new Error("saved: true (the open script as saved) or a .gfr path.") + if (!saved) return withImage(await program(context, "verify", args, VERIFY_TIMEOUT_MS), "verify.png") + // Gaffer only starts the background Gaffer (a moment on its main thread); it is followed here. + const timeout_s = + typeof args.timeout_s === "number" ? Math.min(Math.max(Math.round(args.timeout_s), 5), 3_600) : 540 + const started = (await program(context, "verify_saved", { ...args, timeout_s })) as { started?: unknown } + if (!isSavedStart(started?.started)) throw new Error("gaffer_verify saved: the background Gaffer did not start.") + return withImage(await waitSaved(started.started, { signal: context.signal }), "verify.png") + }, + + async gaffer_render(args, context) { + const { task: id, wait_s, cancel, ...params } = args + if (id !== undefined) { + const task = getTask(String(id)) + if (task && cancel === true) return json(cancelTask(task)) + return json(await follow(String(id), wait_s, context)) + } + if (params.node === undefined) + throw new Error("Give node (a task node to render) to start a task, or task to follow one.") + const started = await program(context, "render", params) + if (!isRenderStart(started)) throw new Error(`gaffer_render: the render did not start: ${json(started)}`) + return json(await follow(addTask(started).id, wait_s, context)) + }, + + async gaffer_save(args, context) { + const result = (await program(context, "save", args)) as Record + const copy = typeof result?.copy === "string" ? result.copy : result?.copy === true ? String(result.path) : null + return json({ ...result, check: verifySavedHint(copy ?? true) }) + }, + } + + const allTools = buildPluginTools( + manifests.list().filter((manifest) => manifest.domain === "tools"), + APP, + async (manifest, args, context) => { + const executor = executors[manifest.name] + if (!executor) throw new Error(`No executor for ${manifest.name}`) + try { + return await executor(args, context) + } catch (error) { + throw hostError(error) + } + }, + ) + + const registrations: Array<{ dispose: () => Promise }> = [] let disconnect = () => {} + let unsubscribe = () => {} const cleanup = async () => { + unsubscribe() disconnect() - for (const item of registrations.splice(0).reverse()) await item.dispose() + for (const registration of registrations.splice(0).reverse()) await registration.dispose().catch(() => {}) } try { - const registration = await ctx.rpc.register(rpc.definition as never, rpc.handlers as never) + await attachments.load() + const registration = await ctx.rpc.register(rpc.definition as never, handlers as never) registrations.push(registration) disconnect = rpc.connect(registration as never) registrations.push( - await ctx.tool.transform((editor) => { - editor.namespace({ name: "gaffer", description: "Inspect and edit the explicitly attached Gaffer script." }) - for (const [name, input] of [ - ["context", contextInput], - ["apply", applyInput], - ] as const) { - editor.add({ - name, - input: input as never, - description: - name === "context" - ? "Read a bounded working set from the attached graph." - : "Apply one atomic revision-checked graph intent with preflight, rollback and verification.", - options: { namespace: "gaffer", codemode: true }, - async execute(raw, tool) { - const args = raw as Record - const ticket = await bindings.binding(tool.sessionID, tool.signal) - if (args.script_handle !== ticket.bound.scriptHandle) - throw new Error("The script handle is not attached to this conversation.") - if (tool.signal.aborted || !registry.current(ticket.authority)) - throw new Error("Attachment changed; read again.") - const result = await requestHost( - ticket.bound.endpoint, - name, - { ...args, session_id: tool.sessionID }, - { signal: tool.signal }, - ) - return { content: resultContent(result as Record) } - }, - }) - } - for (const name of ["hosts", "attach", "detach", "install_host"] as const) { - editor.add({ - name, - description: `${name.replaceAll("_", " ")} for OpenCode for Gaffer.`, - input: - name === "attach" - ? { - type: "object", - properties: { offerID: { type: "string" } }, - required: ["offerID"], - additionalProperties: false, - } - : { type: "object", additionalProperties: false }, - options: { namespace: "gaffer", codemode: true }, - async execute(raw, tool) { - const authority = await bindings.snapshot(tool.sessionID, tool.signal) - if (!authority) throw new Error("Session ownership is unverifiable.") - if (name !== "detach") - await ctx.permission.assert({ - sessionID: tool.sessionID, - agent: tool.agent, - action: name === "install_host" ? "gaffer_install_host" : "gaffer_attach", - resources: ["*"], - source: { type: "tool", messageID: tool.messageID, id: tool.id }, - } as never) - if (!(await bindings.verify(tool.sessionID, authority, tool.signal))) - throw new Error("Attachment changed during permission.") - const result = - name === "hosts" - ? registry.hosts() - : name === "attach" - ? registry.attach(String((raw as { offerID: string }).offerID), tool.sessionID) - : name === "detach" - ? await bindings.detach(tool.sessionID, tool.signal) - : await rpc.handlers.installHost?.({}) - return { content: JSON.stringify(result) } - }, - }) - } - }), - ) - registrations.push( - await ctx.skill.transform((editor) => { - for (const skill of loadSkills()) if (!editor.get(skill.id)) editor.add(skill as never) - }), - ) - registrations.push( - await ctx.session.hook("context", async (event) => { - try { - const { bound } = await bindings.binding(String(event.sessionID)) - event.system.push({ - type: "text", - text: `A Gaffer script is explicitly attached. Use gaffer_context and gaffer_apply with script_handle ${JSON.stringify(bound.scriptHandle)}. Read before editing; use the returned revision and opaque refs.`, - }) - } catch { - /* Unattached sessions receive no graph metadata. */ - } - }), + await ctx.tool.transform((editor) => registerPluginTools(editor, APP, NAMESPACE_DESCRIPTION, allTools)), ) + unsubscribe = registry.onChange(() => void maintain()) + void maintain() } catch (error) { await cleanup() throw error } - return Object.assign(cleanup, { internals: { registry, rpc } }) + return Object.assign(cleanup, { + internals: { registry, rpc, handlers, tools: allTools, executors, options, attachments, maintain }, + }) } diff --git a/integrations/gaffer/plugin/src/legacy.ts b/integrations/gaffer/plugin/src/legacy.ts new file mode 100644 index 0000000..40298a6 --- /dev/null +++ b/integrations/gaffer/plugin/src/legacy.ts @@ -0,0 +1,30 @@ +// Read-only detection of an earlier OpenCode installation in Gaffer (notice `legacy_install`): a startup file +// gaffer/startup/gui/openCode.py not written by this integration, and the gaffer/OpenCode folder it loads, which +// register another OpenCode editor and a package of the same name next to OpenCode Link. Nothing is changed: the +// user removes them. + +import { existsSync, readFileSync, statSync } from "node:fs" +import { join } from "node:path" + +import { HEADER } from "./host-install.ts" +import { gafferUserDir, type PathEnv } from "./paths.ts" + +export type LegacyInstall = { path: string; kind: "startup file" | "folder" } + +export function legacyInstalls(paths: PathEnv, startup?: string): LegacyInstall[] { + const user = gafferUserDir(paths) + const found: LegacyInstall[] = [] + for (const dir of new Set([join(user, "startup", "gui"), ...(startup ? [startup] : [])])) { + const file = join(dir, "openCode.py") + if (!existsSync(file)) continue + try { + if (statSync(file).size <= 1024 * 1024 && readFileSync(file, "utf8").startsWith(HEADER)) continue + } catch { + // unreadable: reported + } + found.push({ path: file, kind: "startup file" }) + } + const folder = join(user, "OpenCode") + if (existsSync(folder)) found.push({ path: folder, kind: "folder" }) + return found +} diff --git a/integrations/gaffer/plugin/src/method.ts b/integrations/gaffer/plugin/src/method.ts new file mode 100644 index 0000000..52a536f --- /dev/null +++ b/integrations/gaffer/plugin/src/method.ts @@ -0,0 +1,26 @@ +// The working method, attached once to a conversation when it attaches a Gaffer script (a native instructions +// entry, not a skill): what to check before writing code, and how to verify a step. + +import { type EntryPut, methodAttacher as kitAttacher, resetMethodForTests as resetKit } from "./kit.ts" + +export const METHOD_KEY = "gaffer.method" +export const METHOD_TEXT = [ + "Working on the attached Gaffer script:", + "- Your knowledge of Gaffer's API may be older than it: read its version (gaffer_hosts, gaffer_inspect).", + "- Inspect the exact element before coding it (gaffer_inspect search, type with its plugs, node, a Python symbol).", + "- After an error, read its hint and inspect again; never guess twice.", + "- Compute what you build (its output plugs) and read its errors; a node whose output fails is not done.", + "- Render with gaffer_render: a background Gaffer runs it, so a renderer crash leaves this Gaffer open.", + "- Turn each requirement of your task into a measured check (gaffer_verify criteria).", + "- The saved file is what you deliver: save, then run every check on it with gaffer_verify saved: true.", + "- Report each requirement with its measured value from the file. A failed or unchecked one is not done: say so.", + "- Still unclear: the Gaffer documentation of THIS version (links in gaffer_inspect).", +].join("\n") + +export type { EntryPut } + +export const methodAttacher = (put: EntryPut | undefined) => kitAttacher(put, { key: METHOD_KEY, text: METHOD_TEXT }) + +export function resetMethodForTests() { + resetKit(METHOD_KEY) +} diff --git a/integrations/gaffer/plugin/src/options.ts b/integrations/gaffer/plugin/src/options.ts new file mode 100644 index 0000000..600595d --- /dev/null +++ b/integrations/gaffer/plugin/src/options.ts @@ -0,0 +1,25 @@ +// Plugin settings: the `options` of the plugin entry first, `NEURALNOMADS_GAFFER_*` variables as a fallback. + +export type GafferOptions = { + // Gaffer's GUI startup folder to inspect and install into, instead of ~/gaffer/startup/gui. + hostStartupDir?: string + // Longest wait for one host request (a tool may give a longer one, such as a verification). + requestTimeoutMs: number +} + +// The installed part waits up to an hour for Gaffer's main thread to finish a request; a call waits this long by +// default, then says whether Gaffer still runs it (status). +const DEFAULT_TIMEOUT_MS = 120_000 + +function text(value: unknown): string | undefined { + return typeof value === "string" && value.trim() ? value.trim() : undefined +} + +export function readOptions(options: Record | undefined, env: NodeJS.ProcessEnv): GafferOptions { + const source = options ?? {} + const timeout = Number(source.requestTimeoutMs ?? env.NEURALNOMADS_GAFFER_REQUEST_TIMEOUT_MS) + return { + hostStartupDir: text(source.hostStartupDir) ?? text(env.NEURALNOMADS_GAFFER_HOST_STARTUP_DIR), + requestTimeoutMs: Number.isFinite(timeout) && timeout >= 1_000 ? Math.min(timeout, 600_000) : DEFAULT_TIMEOUT_MS, + } +} diff --git a/integrations/gaffer/plugin/src/paths.ts b/integrations/gaffer/plugin/src/paths.ts index 171c494..6f9d2f1 100644 --- a/integrations/gaffer/plugin/src/paths.ts +++ b/integrations/gaffer/plugin/src/paths.ts @@ -1,11 +1,32 @@ +// Folders the plugin reads: its own package (methods, programs, host payload), its state folder (the installed +// payload) and Gaffer's user startup folder. + import { homedir } from "node:os" import { join } from "node:path" import { fileURLToPath } from "node:url" +// dist/index.js and src/*.ts both sit one level below the package root. export const packageRoot = fileURLToPath(new URL("..", import.meta.url)) -export const skillsDir = join(packageRoot, "skills") -export function stateRoot() { - return process.platform === "win32" - ? join(process.env.LOCALAPPDATA ?? join(homedir(), "AppData", "Local"), "NeuralNomads", "opencode-gaffer") - : join(process.env.XDG_STATE_HOME ?? join(homedir(), ".local", "state"), "neuralnomads", "opencode-gaffer") +export const methodsDir = join(packageRoot, "methods") +export const programsDir = join(packageRoot, "programs") + +export type PathEnv = { env: NodeJS.ProcessEnv; platform: NodeJS.Platform; home: string } + +export const defaultPathEnv = (): PathEnv => ({ env: process.env, platform: process.platform, home: homedir() }) + +export function stateDir(env = process.env, home = homedir(), platform = process.platform) { + return platform === "win32" + ? join(env.LOCALAPPDATA ?? join(home, "AppData", "Local"), "NeuralNomads", "opencode-gaffer") + : join(env.XDG_STATE_HOME ?? join(home, ".local", "state"), "neuralnomads", "opencode-gaffer") +} + +// Gaffer's user folder: ~/gaffer (GAFFER_STARTUP_PATHS starts with its startup folder; on Windows the home folder +// is USERPROFILE). +export function gafferUserDir({ env, platform, home }: PathEnv) { + return join(platform === "win32" ? env.USERPROFILE || home : env.HOME || home, "gaffer") +} + +// The GUI startup folder the installed part's loader goes into. +export function defaultStartupDir(paths: PathEnv) { + return join(gafferUserDir(paths), "startup", "gui") } diff --git a/integrations/gaffer/plugin/src/plugin.ts b/integrations/gaffer/plugin/src/plugin.ts index 5ccac5b..cf22517 100644 --- a/integrations/gaffer/plugin/src/plugin.ts +++ b/integrations/gaffer/plugin/src/plugin.ts @@ -1,5 +1,4 @@ -import type { Plugin } from "@opencode/plugin" +import { Plugin } from "@opencode/plugin" import { setupGaffer } from "./integration.ts" -const plugin: Plugin.Plugin = { id: "neuralnomads.gaffer", setup: (ctx) => setupGaffer(ctx) } -export default plugin +export default Plugin.define({ id: "neuralnomads.gaffer", setup: setupGaffer }) diff --git a/integrations/gaffer/plugin/src/programs.ts b/integrations/gaffer/plugin/src/programs.ts new file mode 100644 index 0000000..ef4f125 --- /dev/null +++ b/integrations/gaffer/plugin/src/programs.ts @@ -0,0 +1,29 @@ +// The fixed programs of gaffer_inspect, gaffer_verify, gaffer_render and gaffer_save (programs/*.py in the +// package). They run through the host's code execution, read only, with their parameters as JSON data: the +// plugin's own code, never code from the model, so they ask no permission. + +import { readFileSync } from "node:fs" +import { join } from "node:path" + +import { programsDir } from "./paths.ts" + +const read = (file: string) => readFileSync(join(programsDir, file), "utf8") +const PROGRAMS = { + inspect: read("inspect.py"), + verify: read("verify.py"), + verify_saved: read("verify_saved.py"), + save: read("save.py"), + render: read("render.py"), +} +// The background Gaffer's script: render.py's job, and for verify_saved the verify program itself. +const JOBS: Partial> = { + render: read("render_job.py"), + verify_saved: PROGRAMS.verify, +} + +export type ProgramName = keyof typeof PROGRAMS + +export function programCode(name: ProgramName, params: unknown) { + const job = JOBS[name] === undefined ? "" : `JOB = ${JSON.stringify(JOBS[name])}\n` + return `import json\nPARAMS = json.loads(${JSON.stringify(JSON.stringify(params))})\n${job}${PROGRAMS[name]}` +} diff --git a/integrations/gaffer/plugin/src/rpc.ts b/integrations/gaffer/plugin/src/rpc.ts deleted file mode 100644 index 2281399..0000000 --- a/integrations/gaffer/plugin/src/rpc.ts +++ /dev/null @@ -1,4 +0,0 @@ -import { rpcDefinition } from "./kit.ts" - -export const GAFFER_PROTOCOL_VERSION = 1 -export const GafferRpc = rpcDefinition("neuralnomads.gaffer", "native") diff --git a/integrations/gaffer/plugin/src/saved-verify.ts b/integrations/gaffer/plugin/src/saved-verify.ts new file mode 100644 index 0000000..59be622 --- /dev/null +++ b/integrations/gaffer/plugin/src/saved-verify.ts @@ -0,0 +1,163 @@ +// gaffer_verify saved: the verify program runs in a background `gaffer python` started from the open Gaffer, which only +// starts it (programs/verify_saved.py). This module follows it outside Gaffer's main thread: its result, its log's +// steps (OPENCODE_STEP: what it does now), its crash, and timeout_s, after which it is stopped and the answer says +// where it was. + +import { existsSync, readFileSync, rmSync } from "node:fs" + +import { isAlive } from "./kit.ts" +import { crashOf, stopTree } from "./tasks.ts" + +export type SavedStart = { + pid: number + work: string + log: string + result: string + file: string + saved_at: string + bytes: number + timeout_s: number +} + +export function isSavedStart(value: unknown): value is SavedStart { + const start = value as Partial | undefined + return ( + typeof start?.pid === "number" && + typeof start.log === "string" && + typeof start.result === "string" && + typeof start.file === "string" && + typeof start.timeout_s === "number" + ) +} + +type Step = { step?: string; at_s?: number; [key: string]: unknown } + +export function stepsOf(text: string): Step[] { + return text + .split(/\r?\n/) + .filter((line) => line.startsWith("OPENCODE_STEP ")) + .flatMap((line) => { + try { + return [JSON.parse(line.slice("OPENCODE_STEP ".length)) as Step] + } catch { + return [] + } + }) +} + +// What a step says it does, in words. +export function describeStep(step: Step | undefined, file: string) { + if (step?.step === undefined) return "starting (Gaffer had not loaded the file yet)" + switch (step.step) { + case "load": + return `loading ${file}` + case "loaded": + return `loaded ${file} in ${step.seconds} s, before the first criterion` + case "criterion": + return `measuring criterion ${step.index}/${step.of} "${step.label}" at frame ${step.frame}` + case "errors": + return "starting the error scan" + case "cook": + return `computing ${step.node} for the error scan` + case "render": + return "rendering the check image" + case "done": + return "writing its result" + default: + return String(step.step) + } +} + +const read = (file: string) => { + try { + return readFileSync(file, "utf8") + } catch { + return "" + } +} + +export type SavedDeps = { + alive?: (pid: number) => boolean + stop?: (pid: number) => void + now?: () => number + sleep?: (ms: number) => Promise + signal?: AbortSignal +} + +// Waits for the background Gaffer's result, at most start.timeout_s from now. +export async function waitSaved(start: SavedStart, deps: SavedDeps = {}): Promise> { + const alive = deps.alive ?? isAlive + const stop = deps.stop ?? stopTree + const now = deps.now ?? Date.now + const sleep = deps.sleep ?? ((ms: number) => new Promise((done) => setTimeout(done, ms))) + const began = now() + const deadline = began + start.timeout_s * 1_000 + const elapsed = () => Math.round((now() - began) / 100) / 10 + // The background Gaffer holds its log until it has exited (after its result, or after being stopped): removed once it lets go, + // without holding the answer back. + const cleanup = async () => { + for (let attempt = 0; attempt < 100; attempt++) { + try { + rmSync(start.work, { recursive: true, force: true }) + if (!existsSync(start.work)) return + } catch {} + await sleep(200) + } + } + try { + for (;;) { + if (existsSync(start.result)) { + const got = JSON.parse(read(start.result)) as Record + if (typeof got.error === "string") + throw new Error(`verify in the background Gaffer reading ${start.file} failed:\n${got.error}`) + const { load_warnings, load_seconds, ...result } = got + return { + ...result, + read: { + file: start.file, + saved_at: start.saved_at, + bytes: start.bytes, + seconds: elapsed(), + ...(typeof load_seconds === "number" ? { load_seconds } : {}), + by: "a background Gaffer on the file as written on disk, not the open script", + ...(Array.isArray(load_warnings) && load_warnings.length ? { load_warnings } : {}), + }, + } + } + if (!alive(start.pid)) { + // Its result may land just as it exits. + await sleep(300) + if (existsSync(start.result)) continue + const text = read(start.log) + const crash = crashOf(text) + const last = describeStep(stepsOf(text).at(-1), start.file) + throw new Error( + crash + ? `the background Gaffer reading ${start.file} crashed (${crash}) while ${last}; the open Gaffer is not affected. Log: ${start.log}` + : `the background Gaffer reading ${start.file} stopped without a result while ${last}:\n${text.slice(-1200)}`, + ) + } + if (deps.signal?.aborted || now() >= deadline) { + stop(start.pid) + const steps = stepsOf(read(start.log)) + const last = steps.at(-1) + const measured = steps.filter((step) => step.step === "criterion").length + const loaded = steps.find((step) => step.step === "loaded") + throw new Error( + `${deps.signal?.aborted ? "gaffer_verify saved was cancelled" : `gaffer_verify saved stopped the background Gaffer after ${start.timeout_s} s (timeout_s)`}: it was ${describeStep(last, start.file)}. ${loaded ? `The file loaded in ${loaded.seconds} s; ` : ""}${ + measured + ? `${Math.max(0, measured - (last?.step === "criterion" ? 1 : 0))} criteria were measured before. ` + : "" + }${ + last?.step === "criterion" || last?.step === "cook" + ? "Reading frames of a simulation that is not cached simulates them: read its cache (a File SOP), or " + : "" + }give more time with timeout_s (up to 3600).`, + ) + } + await sleep(500) + } + } finally { + void cleanup() + } +} diff --git a/integrations/gaffer/plugin/src/schemas.ts b/integrations/gaffer/plugin/src/schemas.ts deleted file mode 100644 index f1a005c..0000000 --- a/integrations/gaffer/plugin/src/schemas.ts +++ /dev/null @@ -1,131 +0,0 @@ -const string = { type: "string" } as const -const strings = { type: "array", items: string, maxItems: 50 } as const -export const contextInput = { - type: "object", - additionalProperties: false, - required: ["script_handle"], - properties: { - script_handle: string, - if_revision: { type: "integer", minimum: 0 }, - refs: strings, - match: { type: "object", properties: { name: string, type: string }, additionalProperties: false }, - radius: { type: "integer", minimum: 0, maximum: 2 }, - plugs: strings, - plug_limit: { type: "integer", minimum: 1, maximum: 1000 }, - recipe_match: { - type: "object", - properties: { id: string, node_type: string, menu_path: string, variant: string }, - additionalProperties: false, - }, - job: { - type: "object", - properties: { - id: string, - after: { type: "integer", minimum: 0 }, - limit: { type: "integer", minimum: 1, maximum: 200 }, - }, - required: ["id"], - additionalProperties: false, - }, - include: { - type: "array", - items: { enum: ["values", "connections", "schema", "affordances", "capabilities", "recipes", "renders"] }, - uniqueItems: true, - }, - limit: { type: "integer", minimum: 1, maximum: 100 }, - }, -} as const -export const applyInput = { - type: "object", - additionalProperties: false, - required: ["script_handle", "if_revision", "command_id", "operations"], - properties: { - script_handle: string, - if_revision: { type: "integer", minimum: 0 }, - command_id: string, - summary: string, - operations: { - type: "array", - minItems: 1, - maxItems: 50, - items: { - type: "object", - required: ["op"], - additionalProperties: false, - properties: { - op: { - enum: [ - "node.create", - "node.rename", - "plug.set_value", - "graph.connect", - "graph.connect_many", - "graph.delete", - "output.define", - "render.start", - "render.cancel", - "ui.view", - ], - }, - target: string, - source: string, - destination: string, - sources: { type: "array", items: string, minItems: 1, maxItems: 100 }, - start_index: { type: "integer", minimum: 0 }, - value: {}, - type: string, - recipe: string, - variant: string, - name: string, - parent: string, - bind: { type: "string", pattern: "^[A-Za-z_][A-Za-z0-9_]*$" }, - renderer: string, - file_name: string, - driver: string, - channels: string, - frame_range: string, - parameters: { - type: "object", - additionalProperties: { anyOf: [string, { type: "number" }, { type: "boolean" }] }, - }, - }, - }, - }, - expect: { - type: "array", - maxItems: 50, - items: { - oneOf: [ - { - type: "object", - properties: { subject: string, predicate: { const: "exists" }, value: { type: "boolean" } }, - required: ["subject", "predicate"], - additionalProperties: false, - }, - { - type: "object", - properties: { subject: string, predicate: { const: "name_equals" }, value: string }, - required: ["subject", "predicate", "value"], - additionalProperties: false, - }, - { - type: "object", - properties: { subject: string, predicate: { const: "value_equals" }, value: {} }, - required: ["subject", "predicate", "value"], - additionalProperties: false, - }, - { - type: "object", - properties: { - subject: string, - predicate: { const: "connected_to" }, - value: { anyOf: [string, { type: "null" }] }, - }, - required: ["subject", "predicate", "value"], - additionalProperties: false, - }, - ], - }, - }, - }, -} as const diff --git a/integrations/gaffer/plugin/src/service.ts b/integrations/gaffer/plugin/src/service.ts index 9e365e9..29aa3ef 100644 --- a/integrations/gaffer/plugin/src/service.ts +++ b/integrations/gaffer/plugin/src/service.ts @@ -22,8 +22,6 @@ export function serviceFetch(baseUrl: string): typeof globalThis.fetch { if (serviceURL(message.url).origin !== origin) throw new Error("OpenCode service origin changed") const body = message.body ? Buffer.from(await message.arrayBuffer()) : undefined message.signal.throwIfAborted() - // The private Bun sidecar's HTTP adapter ignores proxyEnv; force direct connections there too. - if (process.versions.bun) process.env.no_proxy = process.env.NO_PROXY = "*" return new Promise((resolve, reject) => { const req = request( message.url, diff --git a/integrations/gaffer/plugin/src/sidecar-core.ts b/integrations/gaffer/plugin/src/sidecar-core.ts new file mode 100644 index 0000000..d09aa64 --- /dev/null +++ b/integrations/gaffer/plugin/src/sidecar-core.ts @@ -0,0 +1,375 @@ +// The in-Gaffer panel's conversations: a plain client of the OpenCode service. Gaffer offers each open script by +// itself (the installed part); the panel only attaches its own conversation to its script's offer, and never +// announces or withdraws offers. Credentials stay in this process: SDK and host errors are never forwarded. +import type { OpenCode } from "@opencode/client" +import { rpcDefinition } from "../../../../packages/integration-kit/src/contract.ts" +import { requestHost as kitRequestHost } from "../../../../packages/integration-kit/src/host-client.ts" + +export type Api = ReturnType +type Rpc = Partial< + Record< + "hosts" | "attachHost" | "detachHost" | "bindingRead", + (input: object, options: object) => Promise> + > +> +type ModelRef = { id: string; providerID: string; variant?: string } +type Channel = { + id: string + directory: string + sessionID: string + label: string + offerID: string + host: { host: string; port: number; token: string } + handle: string + abort: AbortController + timer?: NodeJS.Timeout + checking: boolean + tools: Map + model?: ModelRef +} +export type PanelRequest = { id: string; channel: string; method: string; params?: Record } + +// The session metadata naming the Gaffer script a conversation belongs to: Connect takes that one back. +export const SCRIPT_MARKER = "opencode_gaffer_script" + +// A message of this sidecar's own, safe to show in the panel. +class PanelError extends Error {} + +// An error in one line for the panel: its first line, bounded, without a token, an authorization or a port. +export function briefly(error: unknown) { + const text = String((error as { message?: unknown } | null)?.message ?? error ?? "unknown error") + .split(/\r?\n/)[0] + ?.replace(/\b[0-9a-f]{32,}\b/gi, "…") + .replace(/(basic|bearer)\s+\S+/gi, "$1 …") + .replace(/\b(port\s+|:)\d{2,5}\b/gi, "$1…") + .slice(0, 240) + return `${text || "unknown error"}${/[.!?]$/.test(text ?? "") ? "" : "."}` +} + +const modelKey = (model: ModelRef | undefined) => + model ? `${model.providerID}/${model.id}${model.variant ? `#${model.variant}` : ""}` : "" + +type Part = { type?: string; text?: string; name?: string; state?: { status?: string } } +type Message = { type?: string; text?: string; content?: Part[] } +type EventData = { + sessionID?: string + delta?: string + id?: string + name?: string + action?: string + resources?: string[] + model?: ModelRef +} + +function history(messages: ReadonlyArray) { + return messages.flatMap((message) => { + if (message.type === "user") return [{ role: "user", text: message.text ?? "" }] + if (message.type !== "assistant") return [] + return (message.content ?? []).flatMap((part) => { + if (part.type === "text") return [{ role: "assistant", text: part.text ?? "" }] + if (part.type === "tool") return [{ role: "tool", text: `${part.name} / ${part.state?.status ?? "unknown"}` }] + return [] + }) + }) +} + +export function createPanel(deps: { + // The running OpenCode service, or undefined when none is running (never started from here). + discover: () => Promise + send: (message: unknown) => void + requestHost?: typeof kitRequestHost + // How long Connect waits for this script's offer (Gaffer offers each script every few seconds). + offerWaitMs?: number + checkEveryMs?: number +}) { + const requestHost = deps.requestHost ?? kitRequestHost + // A Location opened for the panel loads its plugins first, which took tens of seconds in a trial. + const offerWaitMs = deps.offerWaitMs ?? 30_000 + const checkEveryMs = deps.checkEveryMs ?? 5_000 + const channels = new Map() + let client: Api | undefined + let rpc: Rpc = {} + const event = (c: { id: string }, name: string, data: object) => deps.send({ channel: c.id, event: name, data }) + const options = (c: { directory: string }) => ({ + location: { directory: c.directory }, + signal: AbortSignal.timeout(3000), + }) + const host = (c: Channel, method: "cancel" | "resume") => + requestHost(c.host, method, { script_handle: c.handle, session_id: c.sessionID }, { timeoutMs: 2000 }) + + async function api() { + if (client) return client + const found = await deps.discover() + if (!found) throw new PanelError("Start OpenCode before connecting the panel.") + client = found + rpc = found.rpc(rpcDefinition("neuralnomads.gaffer", "native") as never) as unknown as Rpc + return client + } + + // This script's offer: the one with its label among the offered hosts (offer keys stay private to the service). + async function findOffer(c: { directory: string; label: string }) { + const until = Date.now() + offerWaitMs + for (;;) { + let result: { offers?: unknown } | undefined + try { + result = (await rpc.hosts?.({}, options(c))) as typeof result + } catch (error) { + // The first call in the script's folder: OpenCode opens that Location and loads its plugins meanwhile, and + // the integration's RPC is not there yet (a CodeNomad conversation's own Location is long loaded). + if ( + !/RPC is unavailable/i.test(String((error as { message?: unknown })?.message ?? error)) || + Date.now() > until + ) + throw error + await new Promise((resolve) => setTimeout(resolve, 1_000)) + continue + } + const offers = ((result?.offers ?? []) as Array<{ label: string; offerID: string }>).filter( + (offer) => offer.label === c.label, + ) + if (offers.length > 1) + throw new PanelError(`Several scripts offer ${c.label}: attach this conversation from OpenCode (gaffer_hosts).`) + if (offers[0]) return offers[0] + if (Date.now() > until) throw new PanelError("This script is not offered to OpenCode yet; choose Connect again.") + await new Promise((resolve) => setTimeout(resolve, 1_000)) + } + } + + async function attached(c: Channel) { + const read = await rpc.bindingRead?.({ sessionID: c.sessionID }, options(c)) + return read?.state === "attached" && (read.binding as { offerID?: string } | undefined)?.offerID === c.offerID + } + + // The script's own conversation, when one is marked for it and still open. + async function markedSession(service: Api, directory: string, key: string) { + let cursor: string | undefined + for (let page = 0; page < 10; page += 1) { + const listed = await service.session.list({ directory, order: "desc", limit: 100, cursor }) + const found = listed.data.find( + (candidate) => + !candidate.time?.archived && + (candidate.metadata as Record | undefined)?.[SCRIPT_MARKER] === key, + ) + if (found) return found + cursor = listed.cursor.next ?? undefined + if (!cursor) return undefined + } + return undefined + } + + // Closing releases this panel's conversation only: the offer stays for every other conversation. + async function close(c: Channel) { + if (channels.get(c.id) === c) channels.delete(c.id) + if (c.timer) clearInterval(c.timer) + c.abort.abort() + await host(c, "cancel").catch(() => {}) + // Another client may have moved the conversation to another host: never detach that binding. + await attached(c) + .then((still) => (still ? rpc.detachHost?.({ sessionID: c.sessionID }, options(c)) : undefined)) + .catch(() => {}) + } + + function forward(c: Channel, type: string, data: EventData) { + if (type === "session.model.selected") { + c.model = data.model + event(c, "model", { selected: modelKey(c.model) }) + } else if (type === "session.execution.started") event(c, "status", { state: "busy" }) + else if (type === "session.text.started") event(c, "assistant_start", {}) + else if (type === "session.text.delta") event(c, "assistant_delta", { text: data.delta ?? "" }) + else if (type === "session.text.ended") event(c, "assistant_end", {}) + else if (type === "session.tool.input.started") { + c.tools.set(String(data.id), String(data.name ?? "tool")) + event(c, "tool", { id: data.id, name: data.name, state: "starting" }) + } else if (type === "session.tool.success" || type === "session.tool.failed") { + const name = c.tools.get(String(data.id)) + c.tools.delete(String(data.id)) + event(c, "tool", { id: data.id, name, state: type === "session.tool.success" ? "completed" : "failed" }) + } else if (type === "permission.asked") + event(c, "permission", { request_id: data.id, action: data.action, resources: data.resources ?? [] }) + else if (type === "session.execution.succeeded" || type === "session.idle") event(c, "status", { state: "idle" }) + else if (type === "session.execution.interrupted" || type === "session.execution.failed") { + if (type === "session.execution.failed") event(c, "error", { message: "OpenCode execution failed." }) + event(c, "status", { state: "idle" }) + void host(c, "cancel").catch(() => {}) + } + } + + async function stream(c: Channel, service: Api) { + try { + for await (const item of service.event.subscribe({ signal: c.abort.signal })) { + const data = item.data as unknown as EventData + if (data?.sessionID === c.sessionID) forward(c, item.type, data) + } + } catch { + // ended below + } + if (!c.abort.signal.aborted) event(c, "error", { message: "OpenCode disconnected; choose Connect to reconnect." }) + } + + function watch(c: Channel) { + let warned = false + c.timer = setInterval(() => { + if (c.checking) return + c.checking = true + void api() + .then(() => attached(c)) + .then((still) => { + if (!still) throw new Error("detached") + warned = false + }) + .catch(() => { + client = undefined + if (!warned) + event(c, "error", { message: "Conversation detached or service unavailable; choose Connect to reconnect." }) + warned = true + }) + .finally(() => { + c.checking = false + }) + }, checkEveryMs) + c.timer.unref?.() + } + + // A failed Connect says which step failed and why, in a line (never a credential): the user knows what to check. + const step = async (doing: string, work: () => Promise): Promise => { + try { + return await work() + } catch (error) { + if (error instanceof PanelError) throw error + throw new PanelError( + `Connect failed while ${doing}: ${briefly(error)} Check that OpenCode runs, then choose Connect.`, + ) + } + } + + async function connect(id: string, params: Record, fresh: boolean) { + const previous = channels.get(id) + if (previous) await close(previous) + if (params.host !== "127.0.0.1" || !Number.isInteger(params.port) || !/^[0-9a-f]{64}$/.test(String(params.token))) + throw new Error("Invalid host endpoint.") + const directory = String(params.directory ?? "") + const label = String(params.offer_label ?? "") + const key = String(params.script_key ?? "") + const handle = String(params.script_handle ?? "") + if (!directory || !label || !key || !handle) throw new Error("Incomplete connect request.") + const service = await step("reaching OpenCode", api) + // The script's conversation, unless New asked for another one. + const known = fresh + ? undefined + : await step("finding this script's conversation", () => markedSession(service, directory, key)) + const session = + known ?? + (await step("creating this script's conversation", () => + service.session.create({ + title: `Gaffer: ${String(params.script_name || "untitled")}`, + location: { directory }, + metadata: { [SCRIPT_MARKER]: key }, + }), + )) + const c: Channel = { + id, + directory, + sessionID: session.id, + label, + offerID: "", + host: { host: "127.0.0.1", port: Number(params.port), token: String(params.token) }, + handle, + abort: new AbortController(), + checking: false, + tools: new Map(), + model: (session as { model?: ModelRef }).model, + } + try { + const offer = await step("finding this script's offer", () => findOffer(c)) + const result = await step("attaching the conversation to this script", async () => + rpc.attachHost?.({ offerID: offer.offerID, sessionID: c.sessionID }, options(c)), + ) + if (result?.outcome === "busy") + throw new PanelError("This conversation is attached to another application; detach it there first.") + if (result?.outcome !== "attached") throw new PanelError("Gaffer attachment was refused.") + c.offerID = offer.offerID + channels.set(id, c) + watch(c) + void stream(c, service) + const [messages, models, configured, active] = await step("loading the conversation", () => + Promise.all([ + service.message.list({ sessionID: c.sessionID, order: "desc", limit: 200 }), + service.model.list({ location: { directory } }), + service.model.default({ location: { directory } }), + service.session.active(), + ]), + ) + if (!c.model && configured.data) c.model = configured.data as ModelRef + event(c, "session", { id: c.sessionID, resumed: Boolean(known), model: modelKey(c.model), workspace: directory }) + event(c, "models", { + selected: modelKey(c.model), + models: models.data + .filter((model) => model.enabled) + .map((model) => ({ id: model.id, provider_id: model.providerID, name: model.name })), + }) + event(c, "history", { messages: history([...messages.data].reverse() as never) }) + event(c, "status", { + state: (active as Record)[c.sessionID]?.type === "running" ? "busy" : "idle", + }) + } catch (error) { + await close(c) + throw error + } + } + + async function act(c: Channel, method: string, params: Record) { + if (method === "close") return await close(c) + const service = await api() + if (method === "interrupt") { + await host(c, "cancel").catch(() => {}) + await service.session.interrupt({ sessionID: c.sessionID, resume: false }) + } else if (method === "prompt") { + if (!(await attached(c))) throw new PanelError("Gaffer was detached; choose Connect to attach again.") + await host(c, "resume") + await service.session.prompt({ sessionID: c.sessionID, text: String(params.text ?? "") }) + } else if (method === "switch_model") { + const key = String(params.model ?? "") + const catalog = await service.model.list({ location: { directory: c.directory } }) + const selected = catalog.data.find((model) => model.enabled && `${model.providerID}/${model.id}` === key) + if (!selected) throw new PanelError("This model is not available in this workspace.") + c.model = { id: selected.id, providerID: selected.providerID } + await service.session.switchModel({ sessionID: c.sessionID, model: c.model }) + event(c, "model", { selected: modelKey(c.model) }) + } else if (method === "permission_reply") { + const decision = params.reply + if (decision !== "once" && decision !== "reject") throw new Error("Invalid permission answer.") + await service.permission.reply({ sessionID: c.sessionID, requestID: String(params.request_id), decision }) + } else throw new Error("Unknown panel action.") + } + + async function dispatch(request: PanelRequest) { + try { + const params = request.params ?? {} + if (request.method === "connect" || request.method === "new_session") + await connect(request.channel, params, request.method === "new_session") + else { + const c = channels.get(request.channel) + if (c) await act(c, request.method, params) + else if (request.method !== "close") throw new PanelError("Choose Connect first.") + } + deps.send({ id: request.id, result: {} }) + } catch (error) { + deps.send({ + channel: request.channel, + event: "error", + data: { + message: + error instanceof PanelError + ? error.message + : `The panel action failed: ${briefly(error)} Check that OpenCode runs, then choose Connect.`, + }, + }) + deps.send({ id: request.id, error: { message: "Panel action failed." } }) + } + } + + return { + dispatch, + closeAll: () => Promise.allSettled([...channels.values()].map(close)), + } +} diff --git a/integrations/gaffer/plugin/src/sidecar.ts b/integrations/gaffer/plugin/src/sidecar.ts index 16491fc..2764498 100644 --- a/integrations/gaffer/plugin/src/sidecar.ts +++ b/integrations/gaffer/plugin/src/sidecar.ts @@ -1,599 +1,35 @@ -import { randomUUID } from "node:crypto" -import { homedir } from "node:os" -import { join } from "node:path" +// The in-Gaffer panel's private JSONL client, built into the installed part as sidecar.cjs and run by Node. import readline from "node:readline" - import { OpenCode } from "@opencode/client" - -import { type HostBinding, requestHost } from "./host-client.ts" -import { GafferRpc } from "./rpc.ts" import { discoverService } from "./service.ts" - -type Channel = { - id: string - directory: string - scriptKey: string - sessionID: string - bindingKey: string - offerID?: string - binding: HostBinding - revision: number - lastBindingCheck: number - toolNames: Map - model?: ModelRef -} - -type ModelRef = { - id: string - providerID: string - variant?: string -} - -function modelKey(model: ModelRef | undefined) { - return model ? `${model.providerID}/${model.id}${model.variant ? `#${model.variant}` : ""}` : "" -} - -type Request = { - id?: string - channel?: string - method?: string - params?: Record -} - -let client: ReturnType | undefined -let gafferRpc: any -let eventAbort: AbortController | undefined -const channels = new Map() -const channelQueues = new Map>() -let pollTimer: NodeJS.Timeout | undefined -let pollRunning = false -let restoreTimer: NodeJS.Timeout | undefined - -function emit(message: unknown) { - process.stdout.write(`${JSON.stringify(message)}\n`) -} - -function event(channel: string, name: string, data: Record = {}) { - emit({ event: name, channel, data }) -} - -function errorMessage(error: unknown) { - if (error instanceof Error) return error.message - if (typeof error === "object" && error !== null && "message" in error) { - return String((error as { message: unknown }).message) - } - return String(error) -} - -function errorData(error: unknown) { - const object = typeof error === "object" && error !== null ? (error as Record) : {} - return { - code: typeof object.code === "string" ? object.code : typeof object.type === "string" ? object.type : undefined, - message: errorMessage(error), - data: object.data, - } -} - -function rpcOptions(directory: string, signal?: AbortSignal) { - return { location: { directory }, signal } -} - -async function ensureClient() { - if (client) return client - - // The editor is a client of the user's shared OpenCode2 service. Beta client - // package versions may lag the independently upgraded CLI, so exact-version - // discovery would incorrectly stop the shared service and disrupt all hosts. - const discovered = await discoverService() - const endpoint = discovered - if (!endpoint) throw new Error("OpenCode is not running; start it in your OpenCode client.") - - client = OpenCode.make({ baseUrl: endpoint.url, headers: endpoint.headers, fetch: endpoint.fetch }) - gafferRpc = client.rpc(GafferRpc as never) - startEvents() - return client -} - -function sessionForEvent(input: any) { - return input?.data?.sessionID ?? input?.data?.session_id -} - -function startEvents() { - if (!client || eventAbort) return - const api = client - const controller = new AbortController() - eventAbort = controller - const signal = controller.signal - - void (async () => { - let failure: unknown = new Error("OpenCode event stream closed") - try { - for await (const item of api.event.subscribe({ signal })) { - const sessionID = sessionForEvent(item) - if (!sessionID) continue - for (const channel of channels.values()) { - if (channel.sessionID !== sessionID) continue - forwardOpenCodeEvent(channel, item) - } - } - } catch (error) { - failure = error - } - if (signal.aborted || client !== api) return - - await Promise.allSettled([...channels.values()].map((channel) => cancelHost(channel))) - client = undefined - gafferRpc = undefined - eventAbort = undefined - for (const channel of channels.values()) { - event(channel.id, "status", { state: "reconnecting", message: errorMessage(failure) }) - } - await new Promise((resolve) => setTimeout(resolve, 1000)) - try { - await ensureClient() - await restoreBindings() - } catch (error) { - for (const channel of channels.values()) { - event(channel.id, "error", errorData(error)) - } - } - })() -} - -function forwardOpenCodeEvent(channel: Channel, item: any) { - const data = item.data ?? {} - switch (item.type) { - case "session.model.selected": - channel.model = data.model - event(channel.id, "model", { selected: modelKey(channel.model) }) - break - case "session.execution.started": - event(channel.id, "status", { state: "busy" }) - break - case "session.text.started": - event(channel.id, "assistant_start", { message_id: data.assistantMessageID, ordinal: data.ordinal }) - break - case "session.text.delta": - event(channel.id, "assistant_delta", { - message_id: data.assistantMessageID, - ordinal: data.ordinal, - text: data.delta ?? "", - }) - break - case "session.text.ended": - event(channel.id, "assistant_end", { - message_id: data.assistantMessageID, - ordinal: data.ordinal, - text: data.text ?? "", - }) - break - case "session.tool.input.started": { - channel.toolNames.set(String(data.id), String(data.name ?? "tool")) - event(channel.id, "tool", { - id: data.id, - name: data.name, - message_id: data.assistantMessageID, - state: "starting", - }) - break - } - case "session.tool.success": { - const name = channel.toolNames.get(String(data.id)) - channel.toolNames.delete(String(data.id)) - event(channel.id, "tool", { - id: data.id, - name, - message_id: data.assistantMessageID, - state: "completed", - }) - break - } - case "session.tool.failed": { - const name = channel.toolNames.get(String(data.id)) - channel.toolNames.delete(String(data.id)) - event(channel.id, "tool", { - id: data.id, - name, - message_id: data.assistantMessageID, - state: "failed", - error: data.error?.message, - }) - break - } - case "permission.asked": - event(channel.id, "permission", { - request_id: data.id, - action: data.action, - resources: data.resources ?? [], - message: data.message, - }) - break - case "session.status": - event(channel.id, "status", { state: data.status?.type ?? "unknown" }) - break - case "session.idle": - case "session.execution.succeeded": - event(channel.id, "status", { state: "idle" }) - break - case "session.execution.failed": - event(channel.id, "error", { message: data.error?.message ?? "OpenCode execution failed" }) - event(channel.id, "status", { state: "failed" }) - void cancelHost(channel).catch((error) => event(channel.id, "graph_error", errorData(error))) - break - case "session.execution.interrupted": - event(channel.id, "status", { state: "interrupted" }) - void cancelHost(channel).catch((error) => event(channel.id, "graph_error", errorData(error))) - break - } -} - -function normalizeHistory(messages: any[]) { - return messages.flatMap((message) => { - if (message.type === "user") return [{ role: "user", text: message.text ?? "" }] - if (message.type !== "assistant") return [] - return (message.content ?? []).flatMap((part: any) => { - if (part.type === "text") return [{ role: "assistant", text: part.text ?? "" }] - if (part.type === "tool") { - return [{ role: "tool", text: `${part.name} / ${part.state?.status ?? "unknown"}` }] - } - return [] - }) - }) -} - -async function attachChannel(channelID: string, params: Record, forceNew = false) { - const api = await ensureClient() - const binding: HostBinding = { - host: String(params.host ?? "127.0.0.1"), - port: Number(params.port), - token: String(params.token), - scriptHandle: String(params.script_handle), - } - const directory = String(params.directory) - const scriptKey = String(params.script_key ?? "") - if (!directory || !scriptKey || !binding.scriptHandle || !binding.token || !Number.isInteger(binding.port)) { - throw new Error("connect requires directory, script_key, script_handle, token, and a valid port") - } - - let session: any - if (!forceNew) { - let cursor: string | undefined - for (let page = 0; page < 10 && !session; page += 1) { - const listed = await api.session.list({ directory, order: "desc", limit: 100, cursor }) - session = listed.data.find( - (candidate: any) => !candidate.time?.archived && candidate.metadata?.opencode_gaffer_script === scriptKey, - ) - cursor = listed.cursor.next ?? undefined - if (!cursor) break - } - } - session ??= await api.session.create({ - title: `Gaffer / ${params.script_name ?? "Untitled"}`, - location: { directory }, - metadata: { opencode_gaffer_script: scriptKey }, - }) - - const previous = channels.get(channelID) - const bindingKey = randomUUID() - const channel: Channel = { - id: channelID, - directory, - scriptKey, - sessionID: session.id, - bindingKey, - binding, - revision: 0, - lastBindingCheck: Date.now(), - toolNames: new Map(), - model: session.model, - } - await reattach(channel) - if (previous) { - try { - await stopChannel(previous) - } catch (error) { - await gafferRpc - .detachHost({ sessionID: channel.sessionID }, rpcOptions(channel.directory, AbortSignal.timeout(2000))) - .catch(() => undefined) - throw error - } - } - channels.set(channelID, channel) - - const [history, models, configuredDefault] = await Promise.all([ - api.message.list({ sessionID: session.id, order: "desc", limit: 200 }), - api.model.list({ location: { directory } }), - api.model.default({ location: { directory } }), - ]) - if (!channel.model && configuredDefault.data) channel.model = configuredDefault.data - history.data.reverse() - const active = await api.session.active() - event(channelID, "session", { - id: session.id, - title: session.title, - resumed: !forceNew && normalizeHistory(history.data).length > 0, - revision: channel.revision, - model: modelKey(channel.model), - workspace: directory, - }) - event(channelID, "models", { - selected: modelKey(channel.model), - models: models.data - .filter((model: any) => model.enabled) - .map((model: any) => ({ - id: model.id, - provider_id: model.providerID, - name: model.name, - })), - }) - event(channelID, "history", { messages: normalizeHistory(history.data) }) - event(channelID, "status", { state: active[session.id]?.type === "running" ? "busy" : "idle" }) - startPolling() - return { session_id: session.id, revision: channel.revision } -} - -async function reattach(channel: Channel) { - await advertise(channel) - const listed = await gafferRpc.hosts({}, rpcOptions(channel.directory)) - const offer = listed.offers.find((item: { label: string }) => item.label === offerLabel(channel)) - if (!offer) throw new Error("The Gaffer host offer is unavailable.") - const attached = await gafferRpc.attachHost( - { offerID: offer.offerID, sessionID: channel.sessionID }, - rpcOptions(channel.directory), - ) - if (attached.outcome !== "attached") throw new Error("This conversation could not attach to Gaffer.") - channel.offerID = offer.offerID -} - -function offerLabel(channel: Channel) { - return `Gaffer ${channel.id}`.slice(0, 120) -} -function offerLocation() { - return ( - process.env.NEURALNOMADS_GAFFER_GLOBAL_CONFIG_DIR ?? - join(process.env.XDG_CONFIG_HOME ?? join(homedir(), ".config"), "opencode") - ) -} -async function advertise(channel: Channel) { - const result = await gafferRpc.advertise( - { - host: channel.binding.host, - port: channel.binding.port, - token: channel.binding.token, - script_handle: channel.binding.scriptHandle, - label: offerLabel(channel), - protocol: 1, - offer_key: channel.bindingKey, - }, - rpcOptions(offerLocation()), - ) - if (result.outcome === "refused") throw new Error("The Gaffer host advertisement was refused.") -} - -async function ensureAttached(channel: Channel) { - await advertise(channel) - const status = await gafferRpc.bindingRead({ sessionID: channel.sessionID }, rpcOptions(channel.directory)) - if (status.state !== "attached" || status.binding?.offerID !== channel.offerID) - throw new Error("Gaffer is detached or attached elsewhere; reconnect explicitly in the panel.") - channel.lastBindingCheck = Date.now() -} - -async function cancelHost(channel: Channel) { - await requestHost(channel.binding, "cancel", { session_id: channel.sessionID }, { timeoutMs: 2000 }) -} - -async function cancelHostWithRetry(channel: Channel) { - let failure: unknown - for (let attempt = 0; attempt < 3; attempt += 1) { - try { - await cancelHost(channel) - return - } catch (error) { - failure = error - if (attempt < 2) await new Promise((resolve) => setTimeout(resolve, 250)) - } - } - throw failure -} - -async function stopChannel(channel: Channel) { - try { - // Withdraw only this panel's offer: the conversation may now be attached elsewhere. - await cancelHostWithRetry(channel) - } finally { - await gafferRpc.withdraw( - { offer_key: channel.bindingKey, token: channel.binding.token }, - rpcOptions(offerLocation(), AbortSignal.timeout(2000)), - ) - } -} - -async function restoreBindings() { - let failed = false - for (const channel of channels.values()) { - try { - await advertise(channel) - event(channel.id, "status", { state: "detached", message: "Reconnect explicitly in the panel to attach again." }) - } catch (error) { - failed = true - event(channel.id, "status", { state: "reconnecting", message: errorMessage(error) }) - } - } - if (failed && !restoreTimer) { - restoreTimer = setTimeout(() => { - restoreTimer = undefined - void restoreBindings() - }, 1000) - } -} - -function startPolling() { - if (pollTimer) return - pollTimer = setInterval(() => void pollGraphChanges(), 500) -} - -async function pollGraphChanges() { - if (pollRunning) return - pollRunning = true - try { - for (const channel of channels.values()) { - try { - if (Date.now() - channel.lastBindingCheck >= 5000) { - await ensureAttached(channel) - } - const result = await requestHost( - channel.binding, - "sync", - { after_revision: channel.revision }, - { timeoutMs: 2000 }, - ) - const revision = Number(result.revision ?? channel.revision) - if (revision === channel.revision) continue - if (channels.get(channel.id) !== channel) continue - channel.revision = revision - event(channel.id, "graph_revision", { - revision, - changes: result.changes ?? [], - reset: result.reset === true, - }) - } catch (error) { - event(channel.id, "graph_error", errorData(error)) - } - } - } finally { - pollRunning = false - } -} - -async function dispatch(request: Request) { - const id = request.id - const channelID = String(request.channel ?? "") - const params = request.params ?? {} - try { - let result: unknown - switch (request.method) { - case "connect": - result = await attachChannel(channelID, params) - break - case "new_session": { - const current = channels.get(channelID) - if (!current) throw new Error("Channel is not connected") - result = await attachChannel( - channelID, - { - ...params, - host: current.binding.host, - port: current.binding.port, - token: current.binding.token, - script_handle: current.binding.scriptHandle, - directory: current.directory, - script_key: current.scriptKey, - }, - true, - ) - break - } - case "prompt": { - const channel = channels.get(channelID) - if (!channel) throw new Error("Channel is not connected") - const api = await ensureClient() - await ensureAttached(channel) - await requestHost(channel.binding, "resume", { session_id: channel.sessionID }, { timeoutMs: 2000 }) - result = await api.session.prompt({ sessionID: channel.sessionID, text: String(params.text ?? "") }) - break - } - case "switch_model": { - const channel = channels.get(channelID) - if (!channel) throw new Error("Channel is not connected") - const api = await ensureClient() - const key = String(params.model ?? "") - const catalog = await api.model.list({ location: { directory: channel.directory } }) - const selected = catalog.data.find((model: any) => model.enabled && `${model.providerID}/${model.id}` === key) - if (!selected) throw new Error(`Model is not available in this workspace: ${key}`) - channel.model = { id: selected.id, providerID: selected.providerID } - await api.session.switchModel({ sessionID: channel.sessionID, model: channel.model }) - event(channel.id, "model", { selected: modelKey(channel.model) }) - result = { selected: modelKey(channel.model) } - break - } - case "interrupt": { - const channel = channels.get(channelID) - if (!channel) throw new Error("Channel is not connected") - const api = await ensureClient() - await cancelHost(channel).catch((error) => { - event(channel.id, "graph_error", errorData(error)) - }) - result = await api.session.interrupt({ sessionID: channel.sessionID, resume: false }) - break - } - case "permission_reply": { - const channel = channels.get(channelID) - if (!channel) throw new Error("Channel is not connected") - const api = await ensureClient() - const reply = String(params.reply) - if (reply !== "once" && reply !== "always" && reply !== "reject") { - throw new Error("permission reply must be once, always, or reject") - } - result = await api.permission.reply({ - sessionID: channel.sessionID, - requestID: String(params.request_id), - decision: reply, - }) - break - } - case "close": { - const channel = channels.get(channelID) - if (channel) { - channels.delete(channelID) - if (channels.size === 0 && pollTimer) { - clearInterval(pollTimer) - pollTimer = undefined - } - await ensureClient() - await stopChannel(channel) - } - result = {} - break - } - default: - throw new Error(`Unknown sidecar method: ${request.method}`) - } - if (id !== undefined) emit({ id, result }) - } catch (error) { - if (id !== undefined) emit({ id, error: errorData(error) }) - event(channelID, "error", errorData(error)) - } -} +import { createPanel, type PanelRequest } from "./sidecar-core.ts" + +const send = (value: unknown) => process.stdout.write(`${JSON.stringify(value)}\n`) +const panel = createPanel({ + send, + discover: async () => { + const endpoint = await discoverService() + return endpoint && OpenCode.make({ baseUrl: endpoint.url, headers: endpoint.headers, fetch: endpoint.fetch }) + }, +}) +const queues = new Map>() const input = readline.createInterface({ input: process.stdin, crlfDelay: Infinity }) input.on("line", (line) => { if (!line.trim()) return try { - if (Buffer.byteLength(line, "utf8") > 1024 * 1024) throw new Error("Sidecar request exceeded 1 MiB") - const request = JSON.parse(line) as Request - if (!request.id || !request.channel || !request.method) - throw new Error("Sidecar request requires id, channel, and method") - const previous = channelQueues.get(request.channel) ?? Promise.resolve() - const next = previous.catch(() => undefined).then(() => dispatch(request)) - channelQueues.set(request.channel, next) + if (Buffer.byteLength(line) > 1024 * 1024) throw new Error("Request too large") + const request = JSON.parse(line) as PanelRequest + if (!request.id || !request.channel || !request.method) throw new Error("Invalid request") + const next = (queues.get(request.channel) ?? Promise.resolve()).then(() => panel.dispatch(request)) + queues.set(request.channel, next) void next.finally(() => { - if (channelQueues.get(request.channel!) === next) channelQueues.delete(request.channel!) + if (queues.get(request.channel) === next) queues.delete(request.channel) }) - } catch (error) { - emit({ error: { message: `Malformed sidecar request: ${errorMessage(error)}` } }) + } catch { + send({ error: { message: "Invalid panel request." } }) } }) - input.on("close", () => { - void (async () => { - if (pollTimer) clearInterval(pollTimer) - if (restoreTimer) clearTimeout(restoreTimer) - eventAbort?.abort() - const closingChannels = [...channels.values()] - channels.clear() - if (gafferRpc && client) { - await Promise.allSettled(closingChannels.map((channel) => stopChannel(channel))) - } - process.exit(0) - })() + void panel.closeAll().finally(() => process.exit(0)) }) diff --git a/integrations/gaffer/plugin/src/skills.ts b/integrations/gaffer/plugin/src/skills.ts deleted file mode 100644 index c42dafb..0000000 --- a/integrations/gaffer/plugin/src/skills.ts +++ /dev/null @@ -1,48 +0,0 @@ -// The plugin's skills (skills//SKILL.md), loaded through ctx.skill.transform (PKG-2). - -import { existsSync, readdirSync, readFileSync } from "node:fs" -import { join } from "node:path" - -import { skillsDir } from "./paths.ts" - -export type SkillInfo = { - id: string - name: string - description?: string - autoinvoke?: boolean - path: string - content: string -} - -// A front-matter header of `key: value` lines; `autoinvoke: false` keeps reference skills off the model's list. -export function parseSkill(id: string, file: string, source: string): SkillInfo { - const text = source.replace(/\r\n/g, "\n") - const header = /^---\n([\s\S]*?)\n---\n?/.exec(text) - if (!header) throw new Error(`${id}: SKILL.md has no front-matter header`) - const fields = new Map() - for (const line of (header[1] ?? "").split("\n")) { - const match = /^([a-z][a-z0-9_-]*):\s*(.*)$/i.exec(line) - if (match?.[1]) fields.set(match[1], (match[2] ?? "").trim()) - } - const skill: SkillInfo = { - id, - name: fields.get("name") ?? id, - path: file, - content: text.slice(header[0].length), - } - const description = fields.get("description") - if (description) skill.description = description - if (fields.get("autoinvoke") === "false") skill.autoinvoke = false - return skill -} - -export function loadSkills(root = skillsDir): SkillInfo[] { - if (!existsSync(root)) return [] - return readdirSync(root, { withFileTypes: true }) - .filter((entry) => entry.isDirectory() && existsSync(join(root, entry.name, "SKILL.md"))) - .map((entry) => { - const file = join(root, entry.name, "SKILL.md") - return parseSkill(entry.name, file, readFileSync(file, "utf8")) - }) - .sort((a, b) => a.id.localeCompare(b.id)) -} diff --git a/integrations/gaffer/plugin/src/tasks.ts b/integrations/gaffer/plugin/src/tasks.ts new file mode 100644 index 0000000..4e49f6c --- /dev/null +++ b/integrations/gaffer/plugin/src/tasks.ts @@ -0,0 +1,173 @@ +// Background render tasks started by gaffer_render: a `gaffer python` on this computer, followed from the plugin +// through its log (OPENCODE_PROGRESS, OPENCODE_DONE and OPENCODE_FAILED lines) and the files it writes. Per daemon +// process, as the offers. + +import { execFileSync } from "node:child_process" +import { closeSync, existsSync, openSync, readSync, statSync } from "node:fs" +import { basename } from "node:path" + +import { isAlive } from "./kit.ts" + +export type RenderStart = { + pid: number + log: string + work?: string + kind: "render" + output?: string + targets: string[] + start?: number + end?: number + node: string + source?: string + working_copy?: string + facts?: unknown +} +export type RenderTask = RenderStart & { id: string; started: number; cancelled?: boolean; finished?: number } + +const shared = globalThis as { __neuralnomadsGafferTasks?: Map } +shared.__neuralnomadsGafferTasks ??= new Map() +const tasks = shared.__neuralnomadsGafferTasks +const LOG_TAIL = 256 * 1024 + +export function isRenderStart(value: unknown): value is RenderStart { + const start = value as RenderStart + return ( + typeof start === "object" && + start !== null && + Number.isInteger(start.pid) && + start.pid > 0 && + typeof start.log === "string" && + start.kind === "render" && + Array.isArray(start.targets) + ) +} + +export function addTask(start: RenderStart): RenderTask { + const id = `${start.kind}-${Date.now().toString(36)}-${Math.random().toString(36).slice(2, 6)}` + const task = { ...start, id, started: Date.now() } + tasks.set(id, task) + return task +} + +export const getTask = (id: string) => tasks.get(id) + +function readTail(path: string) { + if (!existsSync(path)) return "" + const size = statSync(path).size + const length = Math.min(size, LOG_TAIL) + const buffer = Buffer.alloc(length) + const fd = openSync(path, "r") + try { + readSync(fd, buffer, 0, length, size - length) + } finally { + closeSync(fd) + } + return buffer.toString("utf8") +} + +function marker(text: string, key: string) { + const prefix = `OPENCODE_${key} ` + const line = text + .split(/\r?\n/) + .filter((entry) => entry.startsWith(prefix)) + .at(-1) + if (!line) return undefined + try { + return JSON.parse(line.slice(prefix.length)) as unknown + } catch { + return line.slice(prefix.length) + } +} + +export function taskStatus(task: RenderTask) { + const text = readTail(task.log) + const done = marker(text, "DONE") as { output?: unknown; files?: unknown } | string | undefined + const failed = marker(text, "FAILED") + const progress = marker(text, "PROGRESS") as { done?: number; total?: number } | undefined + const alive = isAlive(task.pid) + const crash = crashOf(text) + // "done" only when the job said so and every expected file is there, not empty. + const missing = done ? task.targets.filter((file) => !existsSync(file) || statSync(file).size === 0) : [] + const verified = done !== undefined && !missing.length + const state = task.cancelled + ? "cancelled" + : verified + ? "done" + : done !== undefined || failed !== undefined || crash || !alive + ? "failed" + : "rendering" + if (state !== "rendering") task.finished ??= Date.now() + const errors = text + .split(/\r?\n/) + .filter((line) => /\b(Error|ERROR)\b/.test(line) && !line.startsWith("OPENCODE_")) + .slice(-5) + return { + task: task.id, + kind: task.kind, + state, + node: task.node, + ...(task.output ? { output: task.output } : {}), + ...(progress && typeof progress === "object" ? { progress } : {}), + ...(task.start !== undefined ? { frames: [task.start, task.end ?? task.start] } : {}), + elapsed_s: Math.round(((task.finished ?? Date.now()) - task.started) / 100) / 10, + ...(verified && typeof done === "object" ? { result: done } : {}), + ...(state === "failed" + ? { + error: missing.length + ? `the job finished but ${missing.length} of ${task.targets.length} expected file(s) are missing or empty: ${missing + .slice(0, 5) + .map((file) => basename(file)) + .join(", ")}${missing.length > 5 ? ", ..." : ""}` + : typeof failed === "string" + ? failed.slice(-1500) + : failed !== undefined + ? JSON.stringify(failed).slice(-1500) + : crash + ? `the background Gaffer crashed (${crash}); the open Gaffer is not affected.${ + typeof (task.facts as { crash_risk?: unknown } | undefined)?.crash_risk === "string" + ? ` ${(task.facts as { crash_risk: string }).crash_risk}.` + : "" + } See ${task.log}` + : errors.join(" | ") || `the background Gaffer stopped without finishing; see ${task.log}`, + } + : {}), + ...(errors.length && state !== "failed" ? { log_errors: errors } : {}), + ...(task.facts ? { facts: task.facts } : {}), + log: task.log, + ...(task.working_copy ? { working_copy: task.working_copy } : {}), + } +} + +// A process that crashed writes "Fatal error: Segmentation fault" (or "Caught signal N") and its crashing thread's +// frames: the signal and the first functions, innermost first. +export function crashOf(text: string): string | undefined { + const fatal = /Fatal error: ([^\r\n]+)|Caught signal (\d+)/.exec(text) + if (!fatal) return undefined + const what = fatal[1]?.trim() ?? `signal ${fatal[2]}` + const thread = text.slice(text.indexOf("CURRENT THREAD") >= 0 ? text.indexOf("CURRENT THREAD") : fatal.index) + const frames = [...thread.matchAll(/\[([A-Za-z_][\w:~<>]*)\]/g)] + .map((match) => match[1]) + .filter((name, index, all) => name && all.indexOf(name) === index) + .slice(0, 3) + return frames.length ? `${what} in ${frames.join(" < ")}` : what +} + +// A background Gaffer and the processes it started (on Windows, the launcher's cmd.exe runs gaffer.exe, which may +// run a renderer): the whole tree. +export function stopTree(pid: number) { + if (process.platform === "win32") { + try { + execFileSync("taskkill", ["/pid", String(pid), "/T", "/F"], { windowsHide: true, stdio: "ignore" }) + return + } catch {} + } + try { + process.kill(pid) + } catch {} +} + +export function cancelTask(task: RenderTask) { + task.cancelled = true + stopTree(task.pid) + return taskStatus(task) +} diff --git a/integrations/gaffer/plugin/src/zip.ts b/integrations/gaffer/plugin/src/zip.ts deleted file mode 100644 index 7cefbdf..0000000 --- a/integrations/gaffer/plugin/src/zip.ts +++ /dev/null @@ -1 +0,0 @@ -export * from "../../../../packages/integration-kit/src/zip.ts" diff --git a/integrations/gaffer/test/MANUAL.md b/integrations/gaffer/test/MANUAL.md index a2687d5..6fa4102 100644 --- a/integrations/gaffer/test/MANUAL.md +++ b/integrations/gaffer/test/MANUAL.md @@ -2,4 +2,6 @@ All checks use private HOME, USERPROFILE, XDG, TEMP and npm cache directories. Never attach the user's existing scripts, contact a shared OpenCode daemon or install into their Gaffer startup folder. -The automated wire test imports the real `HostServer` and `ScriptState.hello` under stub Gaffer modules; it proves the loopback listener and token gate, not graph semantics. With `GAFFER_TEST_BIN`, the real-host smoke script creates an isolated script, reads a bounded context, applies a node edit, rejects stale revisions, replays an idempotent command and undoes the edit. The portable Gaffer 1.7.6.0 Windows archive runs without a system installation. The launch uses an offscreen Qt platform and private startup paths. Run the panel manually against a private OpenCode service to check streaming, model selection, cancellation, permission answers and explicit reconnect after detach. Renderer plugins and external render executables need their own configured test environment. +The normal suite tests the plugin, the panel's sidecar and the real `HostServer` under stub Gaffer modules. With `GAFFER_TEST_BIN` (Gaffer's launcher), `python/real_host.py` runs OpenCode Link in a real Gaffer with its event loop offscreen: the typed apply, each script's offer and label, `hello` and `status` off the main thread, `execute_code` with its undo step, rollback, hints and quarantine release, and the busy answer within 10 s while another request runs; `programs_gaffer.py` runs the plugin's fixed programs (inspect, verify, saved verify, save, render). The portable Gaffer 1.7.6.0 Windows archive runs without a system installation. + +A windowed check runs Gaffer on a desktop the user never sees, with a private profile whose only OpenCode service is a fake one that records the offers: Gaffer offers its script at startup without any panel. Run the panel manually against a private OpenCode service to check streaming, model selection, cancellation, permission answers and Connect after detach. diff --git a/integrations/gaffer/test/conformance.test.ts b/integrations/gaffer/test/conformance.test.ts index f585b5f..ef4ba3c 100644 --- a/integrations/gaffer/test/conformance.test.ts +++ b/integrations/gaffer/test/conformance.test.ts @@ -1,81 +1,161 @@ import assert from "node:assert/strict" -import { spawn } from "node:child_process" +import { spawn, spawnSync } from "node:child_process" +import { existsSync, mkdtempSync, readdirSync, readFileSync, rmSync } from "node:fs" +import { tmpdir } from "node:os" import { join } from "node:path" import { createInterface } from "node:readline" -import { it } from "node:test" -import { checkCodeExecutionGate, checkHostListener, runSentinelScenario } from "../../../packages/testing/src/index.ts" -import { type HostError, requestHost } from "../plugin/src/kit.ts" -import { loadPlugin } from "./helpers.ts" +import { after, it } from "node:test" +import { type HostError, requestHost } from "../../../packages/integration-kit/src/index.ts" +import { + checkCodeExecutionGate, + checkHostListener, + type ManifestLike, + runSentinelScenario, + startFakeHost, +} from "../../../packages/testing/src/index.ts" +import { readPackage } from "../plugin/src/integration.ts" +import { methodsDir } from "../plugin/src/paths.ts" +import { announcement, loadPlugin } from "./helpers.ts" -it("real Gaffer HostServer: loopback-only, 256-bit token, constant-time authentication and valid hello", async () => { - const child = spawn( - process.env.PYTHON ?? (process.platform === "win32" ? "python" : "python3"), - ["-I", join(import.meta.dirname, "python", "stub_host.py")], - { stdio: ["pipe", "pipe", "pipe"], windowsHide: true }, - ) +const python = process.env.PYTHON ?? (process.platform === "win32" ? "python" : "python3") +const scratch = mkdtempSync(join(tmpdir(), "gaffer-conformance-")) +after(() => rmSync(scratch, { recursive: true, force: true })) + +function manifests(): ManifestLike[] { + return readdirSync(methodsDir, { recursive: true, withFileTypes: true }) + .filter((entry) => entry.isFile() && entry.name.endsWith(".json")) + .map((entry) => JSON.parse(readFileSync(join(entry.parentPath, entry.name), "utf8")) as ManifestLike) +} + +// The installed part's real HostServer under system Python, with only the Gaffer application stubbed. +async function servedHost() { + const child = spawn(python, ["-I", join(import.meta.dirname, "python", "stub_host.py")], { + stdio: ["pipe", "pipe", "inherit"], + windowsHide: true, + }) const line = await new Promise((resolve, reject) => { - createInterface({ input: child.stdout }).once("line", resolve) - child.once("exit", (code) => reject(new Error(`Host exited ${code}`))) - child.once("error", reject) + const timer = setTimeout(() => { + child.kill() + reject(new Error("host startup timed out")) + }, 10_000) + createInterface({ input: child.stdout }).once("line", (line) => { + clearTimeout(timer) + resolve(line) + }) + child.once("exit", () => { + clearTimeout(timer) + reject(new Error("host exited")) + }) }) - const endpoint = JSON.parse(line) + const endpoint = JSON.parse(line) as { host: string; port: number; token: string } + return { + endpoint, + close: async () => { + const stopped = new Promise((resolve) => child.once("exit", resolve)) + child.stdin.end() + await stopped + }, + } +} + +it("the actual Gaffer host transport: authenticated, loopback-only, hello per script, refusals with their data", async () => { + const host = await servedHost() try { - assert.match(endpoint.token, /^[0-9a-f]{64}$/) - assert.deepEqual((await checkHostListener(endpoint, 2_000, { script_handle: "test-script" })).errors, []) - const plugin = await loadPlugin() - try { - assert.deepEqual( - await plugin.rpc("advertise", { - host: "127.0.0.1", - port: endpoint.port, - token: endpoint.token, - script_handle: "test-script", - label: "Gaffer stub", - protocol: 1, - offer_key: "test-offer", - }), - { outcome: "accepted" }, - ) - const hello = (await requestHost(endpoint, "hello", { script_handle: "test-script" })) as { - capabilities: { methods: string[] } - } - const errors = await checkCodeExecutionGate({ - app: "gaffer", - manifests: [], - defaults: { appPreference: false, pluginOption: false }, - probe: async () => ({ - helloMethods: hello.capabilities.methods, - toolNames: [...plugin.tools.keys()], - call: async () => ({ executed: false, permissionsAsked: [], error: "method_disabled" }), - }), - }) - assert.deepEqual(errors, []) - // A refusal keeps the host's message, its data (made JSON-safe) and its own code (data.reason). - await assert.rejects(requestHost(endpoint, "refuse", { script_handle: "test-script" }), (error: HostError) => { - assert.equal(error.code, "invalid_args") - assert.match(error.message, /\(invalid_args\): SceneView renderer is not registered: Arnold /) - assert.deepEqual(error.data?.registered_renderers, ["OpenGL"]) - assert.equal(error.data?.reason, "unsupported_renderer") - assert.equal(typeof error.data?.plug, "string") - return true - }) - } finally { - await plugin.cleanup() + assert.match(host.endpoint.token, /^[0-9a-f]{64}$/) + assert.deepEqual((await checkHostListener(host.endpoint, 2_000, { script_handle: "test-script" })).errors, []) + // hello answers the script it is asked for, and only an open one. + const hello = (await requestHost(host.endpoint, "hello", { script_handle: "test-script" })) as { + script_handle: string } + assert.equal(hello.script_handle, "test-script") + await assert.rejects(requestHost(host.endpoint, "hello", { script_handle: "closed-script" }), /stale_handle/) + // A refusal keeps the host's message, its data (made JSON-safe) and its own code (data.reason). + await assert.rejects(requestHost(host.endpoint, "refuse", { script_handle: "test-script" }), (error: HostError) => { + assert.equal(error.code, "invalid_args") + assert.match(error.message, /\(invalid_args\): SceneView renderer is not registered: Arnold /) + assert.deepEqual(error.data?.registered_renderers, ["OpenGL"]) + assert.equal(error.data?.reason, "unsupported_renderer") + assert.equal(typeof error.data?.plug, "string") + return true + }) } finally { - child.stdin.end() - await new Promise((resolve) => child.once("exit", resolve)) + await host.close() } }) -it("Gaffer RPC sentinel scenario: every output and event is credential-free", async () => { +it("checkCodeExecutionGate: declared dangerous, listed by the host, asked on every call", async () => { + let marker = 0 + const cleanups: Array<() => Promise> = [] + const errors = await checkCodeExecutionGate({ + app: "gaffer", + manifests: manifests(), + async probe() { + // A Gaffer whose execute_code really runs the Python it is given. + const host = await startFakeHost({ + hostVersion: readPackage().version, + appVersion: "1.6.15.0", + capabilities: { + methods: ["hello", "context", "apply", "execute_code"], + operations: [], + undo: "transactional", + background: false, + }, + methods: { + execute_code: (params) => { + const run = spawnSync(python, ["-I", "-c", String(params.code)], { windowsHide: true, encoding: "utf8" }) + return { status: run.status === 0 ? "executed" : "failed", output: run.stdout } + }, + }, + }) + let answer: "allow" | "deny" = "allow" + const plugin = await loadPlugin({ + answer: (action) => { + if (action === "gaffer_execute_code" && answer === "deny") throw new Error("Permission denied") + }, + }) + cleanups.push(async () => { + await plugin.cleanup() + await host.close() + }) + assert.deepEqual(await plugin.rpc("advertise", announcement(host.endpoint, host.scriptHandle)), { + outcome: "accepted", + }) + return { + helloMethods: (host.hello.capabilities as { methods: string[] }).methods, + toolNames: [...plugin.tools.keys()], + async call(permission) { + answer = permission + marker += 1 + const file = join(scratch, `executed-${marker}.txt`) + const code = `open(${JSON.stringify(file)}, "w").write("ran")` + const asked = plugin.permissions.length + let error: string | undefined + try { + await plugin.tool("gaffer_execute_code", { code }) + } catch (thrown) { + error = String(thrown) + } + return { + executed: existsSync(file), + permissionsAsked: plugin.permissions.slice(asked), + ...(error ? { error } : {}), + } + }, + } + }, + }) + for (const cleanup of cleanups.splice(0)) await cleanup() + assert.deepEqual(errors, []) +}) + +it("RPC sentinels stay out of every output and event, including bindingRead", async () => { const plugin = await loadPlugin() try { - const report = await runSentinelScenario( - { call: plugin.rpc, events: plugin.events }, - { sessionID: "ses_test", hostVersion: "0.2.0" }, + const result = await runSentinelScenario( + { call: (method, input) => plugin.rpc(method, input as object), events: plugin.events }, + { sessionID: "ses_test", hostVersion: readPackage().version, appVersion: "1.6.15.0" }, ) - assert.deepEqual(report.errors, []) + assert.deepEqual(result.errors, []) } finally { await plugin.cleanup() } diff --git a/integrations/gaffer/test/gaffer.test.ts b/integrations/gaffer/test/gaffer.test.ts new file mode 100644 index 0000000..29622aa --- /dev/null +++ b/integrations/gaffer/test/gaffer.test.ts @@ -0,0 +1,49 @@ +// The fixed programs (inspect, verify, verify saved, save, render) in a real Gaffer: opt-in, with GAFFER_TEST_BIN +// naming Gaffer's launcher. The run uses private HOME, USERPROFILE, TEMP and startup paths, never the user's. + +import assert from "node:assert/strict" +import { spawnSync } from "node:child_process" +import { mkdirSync, mkdtempSync, rmSync } from "node:fs" +import { tmpdir } from "node:os" +import { join } from "node:path" +import { it } from "node:test" + +it("fixed programs in a real Gaffer (GAFFER_TEST_BIN)", { skip: !process.env.GAFFER_TEST_BIN }, () => { + const scratch = mkdtempSync(join(tmpdir(), "gaffer-programs-")) + mkdirSync(join(scratch, "temp")) + const bin = process.env.GAFFER_TEST_BIN as string + const script = join(import.meta.dirname, "programs_gaffer.py") + try { + const env = { + ...process.env, + HOME: scratch, + USERPROFILE: scratch, + TEMP: join(scratch, "temp"), + TMP: join(scratch, "temp"), + TMPDIR: join(scratch, "temp"), + XDG_CONFIG_HOME: join(scratch, "config"), + XDG_STATE_HOME: join(scratch, "state"), + QT_QPA_PLATFORM: "offscreen", + GAFFER_EXTENSION_PATHS: "", + GAFFER_STARTUP_PATHS: "", + } + const command = process.platform === "win32" ? (process.env.ComSpec ?? "cmd.exe") : bin + const args = process.platform === "win32" ? ["/d", "/s", "/c", `""${bin}" python "${script}""`] : ["python", script] + const run = spawnSync(command, args, { + env, + cwd: scratch, + windowsHide: true, + windowsVerbatimArguments: process.platform === "win32", + encoding: "utf8", + timeout: 900_000, + maxBuffer: 64 * 1024 * 1024, + }) + const report = (run.stdout ?? "") + .split(/\r?\n/) + .filter((line) => / (ok|FAIL) +\d/.test(line) || /passed, \d+ failed/.test(line)) + assert.equal(run.status, 0, `${report.join("\n")}\n${(run.stdout ?? "").slice(-4000)}${run.stderr ?? ""}`) + assert.match(report.at(-1) ?? "", /^\d+ passed, 0 failed/) + } finally { + rmSync(scratch, { recursive: true, force: true }) + } +}) diff --git a/integrations/gaffer/test/helpers.ts b/integrations/gaffer/test/helpers.ts index b4a6989..362ec5a 100644 --- a/integrations/gaffer/test/helpers.ts +++ b/integrations/gaffer/test/helpers.ts @@ -1,40 +1,102 @@ +import { mkdtempSync } from "node:fs" import { tmpdir } from "node:os" -import { join } from "node:path" +import { join, resolve } from "node:path" import type { Plugin } from "@opencode/plugin" import type { Info, ToolContext } from "@opencode/plugin/promise/tool" +import { createOfferRegistry, HELLO_TIMEOUT_MS, requestHost } from "../../../packages/integration-kit/src/index.ts" import { readPackage, setupGaffer } from "../plugin/src/integration.ts" -import { createOfferRegistry } from "../plugin/src/kit.ts" +import { resetMethodForTests } from "../plugin/src/method.ts" +import type { PathEnv } from "../plugin/src/paths.ts" + +// A plugin storage as OpenCode keeps it: JSON values by key, scanned by prefix in key order. +export function memoryStorage() { + const values = new Map() + return { + values, + async get(key: string) { + return values.get(key) + }, + async set(key: string, value: unknown) { + values.set(key, structuredClone(value)) + }, + async remove(key: string) { + values.delete(key) + }, + async scan({ prefix, after, limit = 100 }: { prefix: string; after?: string; limit?: number }) { + const keys = [...values.keys()].filter((key) => key.startsWith(prefix) && (!after || key > after)).sort() + const page = keys.slice(0, limit) + return { + entries: page.map((key) => ({ key, value: values.get(key) })), + ...(keys.length > limit ? { next: page.at(-1) } : {}), + } + }, + } +} + +// A home folder of the test's own: Gaffer's user folder (/gaffer) is looked for there, never in the user's. +export function isolatedPaths(home = mkdtempSync(join(tmpdir(), "gaffer-home-"))): PathEnv { + return { env: {}, platform: process.platform, home } +} + +// What a permission request asked: its action, resources and metadata. +export type Asked = { action: string; resources: string[]; metadata?: Record } export async function loadPlugin( input: { now?: () => number + answer?: (action: string) => Promise | void + options?: object + // NEURALNOMADS_GAFFER_* variables; none by default, so that the user's own never apply. + env?: NodeJS.ProcessEnv + state?: string wsl?: () => boolean - answer?: (action: string) => Promise + paths?: PathEnv + storage?: ReturnType // The daemon's shared offer list, as the plugin creates it, instead of a private one. shared?: boolean } = {}, ) { - const host = readPackage().opencodeIntegration.host! + resetMethodForTests() + const marker = readPackage().opencodeIntegration + const host = marker.host as NonNullable + // As createRegistry: one Gaffer offers each of its scripts, and hello names the offer's script. const registry = createOfferRegistry({ expected: { protocol: host.protocol, range: host.range, appVersions: host.appVersions }, now: input.now, + hello: (endpoint, scriptHandle) => + requestHost(endpoint, "hello", { script_handle: scriptHandle }, { timeoutMs: HELLO_TIMEOUT_MS }), }) - const location = { directory: join(tmpdir(), "gaffer-location") } - const sessions = new Map(["ses_test", "ses_other"].map((id) => [id, { id, location: { ...location } }])) - const tools = new Map() - const handlers = new Map Promise>() + const handlers: Record Promise> = {} const events: Array<{ name: string; data: unknown }> = [] + const tools = new Map() const permissions: string[] = [] - const skills = new Map() + const asked: Asked[] = [] + const methods: Array<{ sessionID: string; key: string }> = [] + const location = { directory: resolve("test-location") } + // The conversations OpenCode knows, by ID, with their Location (a moved one is no longer this Location's). + const sessions = new Map(["ses_test", "ses_other"].map((id) => [id, { id, location: { ...location } }])) + const storage = input.storage ?? memoryStorage() const ctx = { + options: input.options, location, + storage, session: { - get: async ({ sessionID }: { sessionID: string }) => sessions.get(sessionID), - hook: async () => ({ dispose: async () => {} }), + get: async ({ sessionID }: { sessionID: string }) => { + const session = sessions.get(sessionID) + if (!session) throw new Error("not found") + return session + }, + instructions: { + entry: { + put: async ({ sessionID, key }: { sessionID: string; key: string }) => { + methods.push({ sessionID, key }) + }, + }, + }, }, rpc: { - register: async (_definition: unknown, methods: Record Promise>) => { - for (const [key, value] of Object.entries(methods)) handlers.set(key, value) + register: async (_definition: unknown, registered: typeof handlers) => { + Object.assign(handlers, registered) return { events: { emit: async (name: string, data: unknown) => { @@ -46,45 +108,62 @@ export async function loadPlugin( }, }, tool: { - transform: async (callback: (editor: unknown) => void) => { + transform: async (callback: (editor: object) => void) => { callback({ namespace: () => {}, add: (tool: Info) => tools.set(`gaffer_${tool.name}`, tool) }) return { dispose: async () => {} } }, }, - skill: { - transform: async (callback: (editor: unknown) => void) => { - callback({ get: (id: string) => skills.get(id), add: (skill: { id: string }) => skills.set(skill.id, skill) }) - return { dispose: async () => {} } - }, - }, permission: { - assert: async ({ action }: { action: string }) => { - permissions.push(action) - await input.answer?.(action) + assert: async (request: Asked) => { + permissions.push(request.action) + asked.push({ action: request.action, resources: request.resources, metadata: request.metadata }) + await input.answer?.(request.action) }, }, } as unknown as Plugin.Context const cleanup = await setupGaffer(ctx, { registry: input.shared ? undefined : registry, + env: input.env ?? {}, wsl: input.wsl ?? (() => false), + state: input.state, + paths: input.paths ?? isolatedPaths(), + now: input.now, }) const signal = new AbortController().signal return { - cleanup, registry: cleanup.internals.registry, + cleanup, events, permissions, + asked, + methods, + tools, + storage, sessions, location, - tools, - skills, - rpc: (method: string, input: unknown = {}) => handlers.get(method)!(input, { signal }), - tool: (name: string, input: unknown = {}, sessionID = "ses_test") => - tools - .get(name)! - .execute( - input as never, - { sessionID, signal, agent: "build", messageID: "msg_test", id: "call_test" } as ToolContext, - ), + rpc: async (name: string, payload: object = {}) => handlers[name]?.(payload, { signal }), + tool: async (name: string, payload: object = {}, sessionID = "ses_test") => { + const tool = tools.get(name) + if (!tool) throw new Error(`No tool ${name}`) + return tool.execute( + payload as never, + { sessionID, signal, agent: "build", messageID: "msg_test", id: "call_test" } as ToolContext, + ) + }, } } + +// What a tool returned, as JSON. +export function content(result: unknown) { + return JSON.parse(String((result as { content?: unknown })?.content ?? result)) +} + +// An announcement of a fake host's script to the plugin's advertise RPC, as Gaffer's advertiser makes it. +export function announcement( + endpoint: { host: string; port: number; token: string }, + scriptHandle: string, + offerKey = "test-offer", + label = "Gaffer: shot.gfr", +) { + return { ...endpoint, script_handle: scriptHandle, label, protocol: 1, offer_key: offerKey } +} diff --git a/integrations/gaffer/test/panel.test.ts b/integrations/gaffer/test/panel.test.ts new file mode 100644 index 0000000..71abbbc --- /dev/null +++ b/integrations/gaffer/test/panel.test.ts @@ -0,0 +1,392 @@ +import assert from "node:assert/strict" +import { spawnSync } from "node:child_process" +import { cpSync, mkdtempSync, rmSync } from "node:fs" +import { tmpdir } from "node:os" +import { join } from "node:path" +import { it } from "node:test" +import { build } from "esbuild" +import { type Api, createPanel, SCRIPT_MARKER } from "../plugin/src/sidecar-core.ts" + +const python = process.env.PYTHON ?? (process.platform === "win32" ? "python" : "python3") +const KEY = "document:shot" +const HOST = { host: "127.0.0.1", port: 4242, token: "a".repeat(64) } +const CONNECT = { + ...HOST, + protocol: 1, + script_handle: "gaffer:shot", + offer_label: "Gaffer: shot.gfr", + script_key: KEY, + script_name: "shot.gfr", + directory: join(tmpdir(), "gaffer-shots"), +} + +it("the installed panel runs its sidecar with Node: payload layout, Gaffer stubs, no Bun", async () => { + const payload = mkdtempSync(join(tmpdir(), "gaffer-panel-")) + try { + cpSync(join(import.meta.dirname, "..", "host", "python"), join(payload, "python"), { recursive: true }) + // As the host build makes it: one CommonJS file for Node 22 at the payload root. + await build({ + entryPoints: [join(import.meta.dirname, "..", "plugin", "src", "sidecar.ts")], + bundle: true, + format: "cjs", + platform: "node", + target: "node22", + outfile: join(payload, "sidecar.cjs"), + logLevel: "warning", + }) + const sidecar = spawnSync(process.execPath, [join(payload, "sidecar.cjs")], { + input: "not json\n", + encoding: "utf8", + windowsHide: true, + timeout: 20_000, + // Never discover the OpenCode service of the computer running the test. + env: { ...process.env, XDG_STATE_HOME: join(payload, "state") }, + }) + assert.equal(sidecar.status, 0, sidecar.stderr) + assert.deepEqual(JSON.parse(sidecar.stdout), { error: { message: "Invalid panel request." } }) + const probe = spawnSync(python, ["-I", join(import.meta.dirname, "panel_probe.py"), payload], { + encoding: "utf8", + windowsHide: true, + }) + assert.equal(probe.status, 0, probe.stderr) + } finally { + rmSync(payload, { recursive: true, force: true }) + } +}) + +type Item = { type: string; data: Record } +type Input = { offerID: string; sessionID: string } +type Out = { id?: string; event?: string; error?: unknown; data: Record & { message?: string } } + +function fakeService() { + const calls: Array<[string, unknown]> = [] + const bindings = new Map() + const listeners = new Set<(item: Item) => void>() + let offers = [ + { offerID: "offer_other", label: "Gaffer: other.gfr", protocol: "ok" }, + { offerID: "offer_shot", label: "Gaffer: shot.gfr", protocol: "ok" }, + ] + let created = 0 + // The first calls of a Location OpenCode is still opening: the integration's RPC is not there yet. + let unavailable = 0 + const record = + (name: string, result: (input: Input) => unknown = () => ({})) => + async (input: Input) => { + calls.push([name, input]) + return result(input) + } + const rpc = new Proxy({} as Record, { + get: (_target, name: string) => { + const methods: Record unknown> = { + hosts: () => { + if (unavailable > 0) { + unavailable -= 1 + throw new Error("RPC is unavailable: neuralnomads.gaffer") + } + return { offers } + }, + attachHost: ({ offerID, sessionID }) => { + const bound = bindings.get(sessionID) + if (bound && bound !== offerID) return { outcome: "busy" } + bindings.set(sessionID, offerID) + return { outcome: "attached" } + }, + detachHost: ({ sessionID }) => ({ outcome: bindings.delete(sessionID) ? "detached" : "none" }), + bindingRead: ({ sessionID }) => + bindings.has(sessionID) + ? { state: "attached", binding: { offerID: bindings.get(sessionID), label: "x" } } + : { state: "none" }, + } + return record(`rpc.${name}`, methods[name] ?? (() => ({ outcome: "refused" }))) + }, + }) + const api = { + rpc: () => rpc, + session: { + list: record("session.list", () => ({ + data: [ + { id: "ses_archived", metadata: { [SCRIPT_MARKER]: KEY }, time: { archived: 1 } }, + { id: "ses_foreign", metadata: { [SCRIPT_MARKER]: "document:other" }, time: {} }, + { id: "ses_old", metadata: { [SCRIPT_MARKER]: KEY }, time: {} }, + ], + cursor: {}, + })), + create: record("session.create", () => ({ id: `ses_new${++created}` })), + prompt: record("session.prompt"), + interrupt: record("session.interrupt"), + switchModel: record("session.switchModel"), + active: record("session.active", () => ({})), + }, + message: { + list: record("message.list", () => ({ + data: [ + { type: "assistant", content: [{ type: "text", text: "Done." }] }, + { type: "user", text: "Add a sphere" }, + ], + })), + }, + model: { + list: record("model.list", () => ({ + data: [ + { id: "m1", providerID: "p", name: "M1", enabled: true }, + { id: "m2", providerID: "p", name: "M2", enabled: false }, + ], + })), + default: record("model.default", () => ({ data: { id: "m1", providerID: "p" } })), + }, + permission: { reply: record("permission.reply") }, + event: { + subscribe: ({ signal }: { signal: AbortSignal }) => + (async function* () { + const queue: Item[] = [] + let wake = () => {} + const listener = (item: Item) => { + queue.push(item) + wake() + } + listeners.add(listener) + signal.addEventListener("abort", () => wake(), { once: true }) + try { + while (!signal.aborted) { + const item = queue.shift() + if (item) yield item + else + await new Promise((resolve) => { + wake = resolve + }) + } + } finally { + listeners.delete(listener) + } + })(), + }, + } + return { + api: api as unknown as Api, + calls, + bindings, + setOffers: (next: typeof offers) => { + offers = next + }, + setUnavailable: (calls: number) => { + unavailable = calls + }, + push: async (item: Item) => { + for (const listener of listeners) listener(item) + await new Promise((resolve) => setImmediate(resolve)) + }, + } +} + +function harness(offerWaitMs = -1) { + const service = fakeService() + const out: Out[] = [] + const host: Array<[string, unknown]> = [] + const panel = createPanel({ + discover: async () => service.api, + send: (message) => out.push(message as Out), + requestHost: async (_endpoint, method, params) => { + host.push([method, params]) + return {} + }, + offerWaitMs, + checkEveryMs: 60_000, + }) + let id = 0 + const call = (method: string, params: Record = {}, channel = "panel") => + panel.dispatch({ id: String(++id), channel, method, params }) + const events = (name: string) => out.filter((message) => message.event === name).map((message) => message.data) + return { service, out, host, panel, call, events } +} + +it("Connect resumes the script's marked conversation and attaches it to that script's offer only", async () => { + const { service, host, panel, call, events } = harness() + try { + await call("connect", CONNECT) + assert.equal(service.calls.filter(([name]) => name === "session.create").length, 0) + assert.deepEqual(service.bindings, new Map([["ses_old", "offer_shot"]])) + assert.deepEqual(events("session")[0], { + id: "ses_old", + resumed: true, + model: "p/m1", + workspace: CONNECT.directory, + }) + assert.deepEqual(events("history")[0]?.messages, [ + { role: "user", text: "Add a sphere" }, + { role: "assistant", text: "Done." }, + ]) + assert.deepEqual(events("models")[0], { selected: "p/m1", models: [{ id: "m1", provider_id: "p", name: "M1" }] }) + assert.deepEqual(events("status").at(-1), { state: "idle" }) + + // Streaming: this conversation's events only. + await service.push({ type: "session.text.delta", data: { sessionID: "ses_other", delta: "no" } }) + await service.push({ type: "session.execution.started", data: { sessionID: "ses_old" } }) + await service.push({ type: "session.text.delta", data: { sessionID: "ses_old", delta: "Hel" } }) + await service.push({ type: "session.text.delta", data: { sessionID: "ses_old", delta: "lo" } }) + await service.push({ + type: "permission.asked", + data: { sessionID: "ses_old", id: "per_1", action: "gaffer_write" }, + }) + assert.deepEqual(events("assistant_delta"), [{ text: "Hel" }, { text: "lo" }]) + assert.deepEqual(events("status").at(-1), { state: "busy" }) + assert.deepEqual(events("permission"), [{ request_id: "per_1", action: "gaffer_write", resources: [] }]) + + await call("permission_reply", { request_id: "per_1", reply: "once" }) + await call("prompt", { text: "Add a light" }) + await call("switch_model", { model: "p/m1" }) + await call("interrupt") + const names = service.calls.map(([name]) => name) + assert.deepEqual(service.calls.find(([name]) => name === "permission.reply")?.[1], { + sessionID: "ses_old", + requestID: "per_1", + decision: "once", + }) + assert.deepEqual(service.calls.find(([name]) => name === "session.prompt")?.[1], { + sessionID: "ses_old", + text: "Add a light", + }) + assert.ok(names.includes("session.switchModel") && names.includes("session.interrupt")) + assert.deepEqual(host, [ + ["resume", { script_handle: "gaffer:shot", session_id: "ses_old" }], + ["cancel", { script_handle: "gaffer:shot", session_id: "ses_old" }], + ]) + + // New: another conversation, marked for the same script, on the same offer; the previous one is released. + await call("new_session", CONNECT) + const created = service.calls.find(([name]) => name === "session.create")?.[1] + assert.deepEqual(created, { + title: "Gaffer: shot.gfr", + location: { directory: CONNECT.directory }, + metadata: { [SCRIPT_MARKER]: KEY }, + }) + assert.deepEqual(service.bindings, new Map([["ses_new1", "offer_shot"]])) + assert.deepEqual(events("session").at(-1), { + id: "ses_new1", + resumed: false, + model: "p/m1", + workspace: CONNECT.directory, + }) + + // Close: this conversation only; the offer is never withdrawn and never announced by the panel. + await call("close") + assert.deepEqual(service.bindings, new Map()) + assert.deepEqual(host.at(-1), ["cancel", { script_handle: "gaffer:shot", session_id: "ses_new1" }]) + await call("prompt", { text: "late" }) + assert.equal(events("error").at(-1)?.message, "Choose Connect first.") + const rpcMethods = new Set(service.calls.map(([name]) => name).filter((name) => name.startsWith("rpc."))) + assert.deepEqual(rpcMethods, new Set(["rpc.hosts", "rpc.attachHost", "rpc.bindingRead", "rpc.detachHost"])) + } finally { + await panel.closeAll() + } +}) + +it("Connect refuses an ambiguous or absent offer, keeps a conversation attached elsewhere and hides SDK errors", async () => { + const { service, panel, call, events } = harness() + try { + service.setOffers([ + { offerID: "offer_a", label: "Gaffer: shot.gfr", protocol: "ok" }, + { offerID: "offer_b", label: "Gaffer: shot.gfr", protocol: "ok" }, + ]) + await call("connect", CONNECT) + assert.match(String(events("error").at(-1)?.message), /^Several scripts offer Gaffer: shot.gfr/) + service.setOffers([]) + await call("connect", CONNECT) + assert.equal(events("error").at(-1)?.message, "This script is not offered to OpenCode yet; choose Connect again.") + assert.equal(service.calls.filter(([name]) => name === "rpc.attachHost").length, 0) + + // The marked conversation is attached to another application: never taken from it. + service.setOffers([ + { offerID: "offer_shot", label: "Gaffer: shot.gfr", protocol: "ok" }, + { offerID: "offer_blender", label: "Blender: shot.blend", protocol: "ok" }, + ]) + service.bindings.set("ses_old", "offer_blender") + await call("connect", CONNECT) + assert.match(String(events("error").at(-1)?.message), /attached to another application/) + assert.equal(service.bindings.get("ses_old"), "offer_blender") + + // Attached here, then moved elsewhere by another client: closing leaves that binding alone. + service.bindings.clear() + await call("connect", CONNECT) + service.bindings.set("ses_old", "offer_blender") + await call("close") + assert.equal(service.bindings.get("ses_old"), "offer_blender") + + for (const bad of [{ token: "short" }, { host: "10.0.0.1" }, { offer_label: "" }]) { + await call("connect", { ...CONNECT, ...bad }) + assert.match( + String(events("error").at(-1)?.message), + /^The panel action failed: (Invalid host endpoint|Incomplete connect request)\. Check that OpenCode runs/, + ) + } + service.bindings.clear() + await call("connect", CONNECT) + ;(service.api.session as { prompt: unknown }).prompt = async () => { + throw new Error(`token ${HOST.token} on port ${HOST.port}`) + } + await call("prompt", { text: "x" }) + const message = events("error").at(-1)?.message as string + assert.ok(!message.includes(HOST.token) && !message.includes(String(HOST.port)), message) + } finally { + await panel.closeAll() + } +}) + +// Real trial: the first Connect failed while OpenCode opened the script's folder (its Location) and loaded its plugins +// ("RPC is unavailable"); it was read as the CodeNomad conversation attached to the same script holding it. +it("Connect waits for the Location OpenCode is still opening, next to a conversation attached to the same script", async () => { + const { service, panel, call, events } = harness(10_000) + try { + service.bindings.set("ses_codenomad", "offer_shot") + service.setUnavailable(2) + await call("connect", CONNECT) + assert.deepEqual(events("error"), []) + assert.equal(service.calls.filter(([name]) => name === "rpc.hosts").length, 3) + // Both conversations are attached to the script; closing the panel leaves the other one attached. + assert.deepEqual( + service.bindings, + new Map([ + ["ses_codenomad", "offer_shot"], + ["ses_old", "offer_shot"], + ]), + ) + await call("close") + assert.deepEqual(service.bindings, new Map([["ses_codenomad", "offer_shot"]])) + } finally { + await panel.closeAll() + } +}) + +it("a failed Connect says which step failed and why, without a credential", async () => { + const { service, panel, call, events } = harness(-1) + try { + service.setUnavailable(1) + await call("connect", CONNECT) + assert.equal( + events("error").at(-1)?.message, + "Connect failed while finding this script's offer: RPC is unavailable: neuralnomads.gaffer. Check that OpenCode runs, then choose Connect.", + ) + ;(service.api.session as { list: unknown }).list = async () => { + throw new Error(`refused by 127.0.0.1:${HOST.port} with Basic secret ${HOST.token}\nstack`) + } + await call("connect", CONNECT) + const message = String(events("error").at(-1)?.message) + assert.match( + message, + /^Connect failed while finding this script's conversation: refused by 127\.0\.0\.1:… with Basic …/, + ) + assert.ok( + !message.includes(HOST.token) && !message.includes(String(HOST.port)) && !message.includes("stack"), + message, + ) + } finally { + await panel.closeAll() + } +}) + +it("Connect without a running OpenCode says so", async () => { + const out: Out[] = [] + const panel = createPanel({ discover: async () => undefined, send: (message) => out.push(message as never) }) + await panel.dispatch({ id: "1", channel: "panel", method: "connect", params: CONNECT }) + assert.equal(out[0]?.data.message, "Start OpenCode before connecting the panel.") + assert.deepEqual(out[1], { id: "1", error: { message: "Panel action failed." } }) +}) diff --git a/integrations/gaffer/test/panel_probe.py b/integrations/gaffer/test/panel_probe.py new file mode 100644 index 0000000..33bda18 --- /dev/null +++ b/integrations/gaffer/test/panel_probe.py @@ -0,0 +1,296 @@ +"""Exercise the installed Gaffer panel (OpenCodeEditor and its sidecar client) without Gaffer or Qt. + +Usage: python -I panel_probe.py +""" +import gc +import hashlib +import pathlib +import sys +import types +import unittest +import weakref +from unittest.mock import MagicMock, patch + + +PAYLOAD = pathlib.Path(sys.argv[1]).resolve() +HOST = {"host": "127.0.0.1", "port": 4242, "token": "a" * 64, "protocol": 1} + + +def widget(*args, **kw): + return MagicMock() + + +def weak_method(method): + # Gaffer.WeakMethod: callbacks never keep an editor alive. + ref = weakref.WeakMethod(method) + return lambda *args, **kw: ref()(*args, **kw) + + +class Editor: + def __init__(self, topLevelWidget, scriptNode, **kw): + self.__script = scriptNode + self.__widget = MagicMock() + + def scriptNode(self): + return self.__script + + def _qtWidget(self): + return self.__widget + + @staticmethod + def registerType(name, cls): + Editor.registered = (name, cls) + + +class Api(types.ModuleType): + """GafferOpenCode as the panel may use it: anything outside the contract fails the probe.""" + + CONTRACT = ("hostInfo", "offerInfo", "scriptHandle", "acquirePanel", "releasePanel") + + def __init__(self): + super().__init__("GafferOpenCode") + self.calls = [] + self.offers = {} + + def __getattr__(self, name): + raise AssertionError("the panel used GafferOpenCode." + name) + + def hostInfo(self): + return dict(HOST) + + def offerInfo(self, script): + return dict(self.offers[script.handle]) + + def scriptHandle(self, script): + return script.handle + + def acquirePanel(self, channel): + self.calls.append(("acquire", channel)) + + def releasePanel(self, channel): + self.calls.append(("release", channel)) + + +class Plug: + def __init__(self, value): + self.value = value + + def getValue(self): + return self.value + + def isSame(self, other): + return self is other + + +class Script: + def __init__(self, handle, fileName=""): + self.handle = handle + self.plugs = {"fileName": Plug(fileName)} + self.slots = [] + + def __getitem__(self, name): + return self.plugs[name] + + def getName(self): + return "ScriptNode" + + def plugSetSignal(self): + script = self + + class Signal: + def connect(self, slot, scoped=True): + script.slots.append(slot) + return object() + + return Signal() + + def setFileName(self, value): + self.plugs["fileName"].value = value + for slot in self.slots: + slot(self.plugs["fileName"]) + + +class FakeClient: + def __init__(self): + self.callbacks = {} + self.sent = [] + self.closed = [] + + def register(self, callback): + channel = "channel-" + str(len(self.callbacks)) + self.callbacks[channel] = weakref.WeakMethod(callback) + return channel + + def send(self, channel, method, params): + self.sent.append((channel, method, params)) + + def close(self, channel): + self.closed.append(channel) + + def emit(self, channel, name, data): + self.callbacks[channel]()(name, data) + + +api = Api() +gaffer = types.ModuleType("Gaffer") +gaffer.WeakMethod = weak_method +gafferui = types.ModuleType("GafferUI") +gafferui.Editor = Editor +for name in ("ListContainer", "Label", "MenuButton", "Menu", "Button", "Spacer", "MultiLineTextWidget"): + setattr(gafferui, name, MagicMock(side_effect=widget)) +qt = types.ModuleType("Qt") +qt.QtCore = MagicMock() +qt.QtGui = MagicMock() +sys.modules.update( + { + "Gaffer": gaffer, + "GafferUI": gafferui, + "GafferOpenCode": api, + "IECore": MagicMock(), + "imath": MagicMock(), + "Qt": qt, + } +) +sys.path.insert(0, str(PAYLOAD / "python")) +import GafferOpenCodeUI # noqa: E402 + +editor_module = sys.modules["GafferOpenCodeUI.OpenCodeEditor"] +client_module = sys.modules["GafferOpenCodeUI.Client"] + + +def private(editor, name): + return getattr(editor, "_OpenCodeEditor__" + name) + + +class PanelTests(unittest.TestCase): + def setUp(self): + api.calls.clear() + self.client = FakeClient() + patcher = patch.object(editor_module.ClientProcess, "acquire", return_value=self.client) + patcher.start() + self.addCleanup(patcher.stop) + + def test_installed_client_runs_node_sidecar(self): + self.assertEqual(Editor.registered, ("OpenCode for Gaffer", GafferOpenCodeUI.OpenCodeEditor)) + process = qt.QtCore.QProcess.return_value + process.state.return_value = qt.QtCore.QProcess.NotRunning + with patch.dict("os.environ", {"NEURALNOMADS_GAFFER_NODE": ""}), patch.object( + client_module.shutil, "which", return_value="/usr/bin/node" + ) as which: + client_module.ClientProcess() + which.assert_called_with("node") + process.setProgram.assert_called_with("/usr/bin/node") + process.setArguments.assert_called_with([str(PAYLOAD / "sidecar.cjs")]) + process.start.assert_called() + self.assertTrue((PAYLOAD / "sidecar.cjs").is_file()) + # NEURALNOMADS_GAFFER_NODE names Node explicitly. + with patch.dict("os.environ", {"NEURALNOMADS_GAFFER_NODE": "/opt/node22/bin/node"}): + client_module.ClientProcess() + process.setProgram.assert_called_with("/opt/node22/bin/node") + + def test_connect_targets_this_scripts_offer_and_closing_never_withdraws(self): + shot = pathlib.Path(PAYLOAD / "shots" / "shot.gfr") + script = Script("gaffer:shot", str(shot)) + api.offers[script.handle] = {"label": "Gaffer: shot.gfr", "offer_key": "gaffer-shot", "advertising": True} + editor = GafferOpenCodeUI.OpenCodeEditor(script) + # Opening the panel attaches nothing: Gaffer already offers the script by itself. + self.assertEqual(api.calls, []) + self.assertEqual(self.client.sent, []) + + private(editor, "toggleConnection")() + channel, method, params = self.client.sent[-1] + self.assertEqual(api.calls, [("acquire", channel)]) + self.assertEqual(method, "connect") + key = "document:" + hashlib.sha256(str(shot.resolve()).encode("utf-8")).hexdigest() + self.assertEqual( + params, + { + **HOST, + "script_handle": "gaffer:shot", + "offer_label": "Gaffer: shot.gfr", + "script_key": key, + "script_name": "shot.gfr", + "directory": str(shot.resolve().parent), + }, + ) + self.client.emit(channel, "session", {"id": "ses_shot", "resumed": True}) + self.client.emit(channel, "status", {"state": "idle"}) + prompt = private(editor, "prompt") + prompt.getText.return_value = "Add a sphere" + private(editor, "send")(None) + self.assertEqual(self.client.sent[-1], (channel, "prompt", {"text": "Add a sphere"})) + + # New starts another conversation for the same script, on the same offer. + private(editor, "newSession")(None) + self.assertEqual(self.client.sent[-1], (channel, "new_session", params)) + + # Disconnect releases the panel's conversation only. + private(editor, "toggleConnection")() + self.assertEqual(self.client.sent[-1], (channel, "close", {})) + self.assertEqual(api.calls[-1], ("release", channel)) + # A reply to the closed connection is ignored. + self.client.emit(channel, "session", {"id": "ses_late"}) + self.assertFalse(private(editor, "connected")) + + # Connect again asks for the same script's conversation: the sidecar resumes the session marked with it. + private(editor, "toggleConnection")() + self.assertEqual(self.client.sent[-1], (channel, "connect", params)) + + # Save As: another file is another script's conversation. + moved = PAYLOAD / "shots" / "shot_v2.gfr" + api.offers[script.handle]["label"] = "Gaffer: shot_v2.gfr" + script.setFileName(str(moved)) + self.assertEqual(self.client.sent[-1], (channel, "close", {})) + self.assertEqual(api.calls[-1], ("release", channel)) + private(editor, "toggleConnection")() + moved_params = self.client.sent[-1][2] + self.assertEqual(moved_params["offer_label"], "Gaffer: shot_v2.gfr") + self.assertNotEqual(moved_params["script_key"], key) + + # Closing the editor (its last reference gone) releases its conversation and the panel registration. + del editor, prompt + gc.collect() + self.assertEqual(self.client.closed, [channel]) + self.assertEqual(api.calls[-1], ("release", channel)) + self.assertEqual({call[0] for call in api.calls}, {"acquire", "release"}) + + def test_each_script_has_its_own_offer_and_conversation(self): + first = Script("gaffer:one") + second = Script("gaffer:two") + api.offers[first.handle] = {"label": "Gaffer: untitled", "offer_key": "gaffer-one", "advertising": True} + api.offers[second.handle] = {"label": "Gaffer: untitled (2)", "offer_key": "gaffer-two", "advertising": True} + editors = [GafferOpenCodeUI.OpenCodeEditor(first), GafferOpenCodeUI.OpenCodeEditor(second)] + for editor in editors: + private(editor, "toggleConnection")() + (a, _, one), (b, _, two) = self.client.sent + self.assertNotEqual(a, b) + self.assertEqual((one["offer_label"], one["script_key"]), ("Gaffer: untitled", "untitled:gaffer:one")) + self.assertEqual((two["offer_label"], two["script_key"]), ("Gaffer: untitled (2)", "untitled:gaffer:two")) + # A second panel on the same script takes back the same conversation. + third = GafferOpenCodeUI.OpenCodeEditor(first) + private(third, "toggleConnection")() + self.assertEqual(self.client.sent[-1][2]["script_key"], "untitled:gaffer:one") + # Closing one panel leaves the others (and every offer) alone. + del third + gc.collect() + self.assertEqual(self.client.closed, ["channel-2"]) + self.assertEqual(api.calls[-1], ("release", "channel-2")) + self.assertEqual([call for call in api.calls if call[0] == "release"], [("release", "channel-2")]) + + def test_a_script_not_offered_is_said_without_connecting(self): + script = Script("gaffer:private") + api.offers[script.handle] = {"label": "Gaffer: untitled", "offer_key": "gaffer-private", "advertising": False} + editor = GafferOpenCodeUI.OpenCodeEditor(script) + append = MagicMock() + setattr(editor, "_OpenCodeEditor__append", append) + private(editor, "toggleConnection")() + self.assertEqual(self.client.sent, []) + self.assertEqual(api.calls, []) + append.assert_called_once_with("error", "OpenCode Link does not offer this script to OpenCode.") + self.assertFalse(private(editor, "active")) + + +if __name__ == "__main__": + suite = unittest.defaultTestLoader.loadTestsFromTestCase(PanelTests) + result = unittest.TextTestRunner(verbosity=2).run(suite) + sys.exit(0 if result.wasSuccessful() else 1) diff --git a/integrations/gaffer/test/plugin.test.ts b/integrations/gaffer/test/plugin.test.ts index 00fac13..c066688 100644 --- a/integrations/gaffer/test/plugin.test.ts +++ b/integrations/gaffer/test/plugin.test.ts @@ -1,159 +1,453 @@ +// OpenCode for Gaffer 0.3.0 against a fake installed part: the tools are the manifests', the conversation attaches +// itself to the only free script, the working method joins it, code and changes run only with permission, an +// earlier installation is said, and OpenCode Link installs into Gaffer's startup folder. + import assert from "node:assert/strict" import { createHash } from "node:crypto" -import { existsSync, mkdtempSync, readdirSync, readFileSync, rmSync, writeFileSync } from "node:fs" +import { existsSync, mkdirSync, mkdtempSync, readdirSync, readFileSync, rmSync, writeFileSync } from "node:fs" import { tmpdir } from "node:os" import { join } from "node:path" -import { it } from "node:test" +import { describe, it } from "node:test" + +import { readStoredZip } from "../../../packages/integration-kit/src/zip.ts" import { startFakeHost } from "../../../packages/testing/src/index.ts" -import { installHost } from "../plugin/src/host-install.ts" -import { readPackage, resultContent } from "../plugin/src/integration.ts" -import { packageRoot, stateRoot } from "../plugin/src/paths.ts" -import { readStoredZip } from "../plugin/src/zip.ts" -import { loadPlugin } from "./helpers.ts" - -it("Gaffer attaches only owned sessions; revision, authority, expiry, detach and move fencing", async () => { - let now = 10_000 - const plugin = await loadPlugin({ now: () => now }) - const host = await startFakeHost({ hostVersion: "0.2.0" }) - try { - const announced = { - host: "127.0.0.1", - port: host.endpoint.port, - token: host.endpoint.token, - script_handle: host.scriptHandle, - label: "Gaffer test", - protocol: 1, - offer_key: "test-offer", - } - assert.deepEqual(await plugin.rpc("advertise", { ...announced, host: "localhost" }), { outcome: "refused" }) - assert.deepEqual(await plugin.rpc("advertise", { ...announced, token: "0".repeat(64) }), { outcome: "refused" }) - assert.deepEqual(await plugin.rpc("advertise", announced), { outcome: "accepted" }) - const offer = plugin.registry.hosts().offers[0]! - assert.deepEqual(await plugin.rpc("attachHost", { offerID: offer.offerID, sessionID: "ses_foreign" }), { - outcome: "refused", - }) - assert.deepEqual(await plugin.rpc("bindingRead", { sessionID: "ses_foreign" }), { state: "unknown" }) - const before = (await plugin.rpc("bindingRead", { sessionID: "ses_test" })) as { - state: string - authority: { revision: number } - } - assert.equal(before.state, "none") - assert.deepEqual(await plugin.rpc("attachHost", { offerID: offer.offerID, sessionID: "ses_test" }), { - outcome: "attached", +import { HEADER, inspectHost, LOADER } from "../plugin/src/host-install.ts" +import { readPackage, resultContent, verifySavedHint } from "../plugin/src/integration.ts" +import { METHOD_KEY } from "../plugin/src/method.ts" +import { methodsDir, packageRoot } from "../plugin/src/paths.ts" +import { announcement, content, isolatedPaths, loadPlugin } from "./helpers.ts" + +const version = readPackage().version +const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)) + +function capabilities(methods: string[]) { + return { + methods: [...new Set(["hello", "context", "apply", "status", "release", ...methods])], + operations: [], + undo: "transactional" as const, + background: false, + } +} + +async function offered( + methods: Record) => unknown> = {}, + process?: { pid: number; temp_dir: string }, +) { + return await startFakeHost({ + hostVersion: version, + appVersion: "1.6.15.0", + capabilities: capabilities(Object.keys(methods)), + methods: { context: () => ({ revision: 3, nodes: [] }), ...methods }, + ...(process ? { helloOverride: (hello: Record) => ({ ...hello, process }) } : {}), + }) +} + +describe("OpenCode for Gaffer 0.3.0", () => { + it("registers one tool per manifest, under its manifest name", async () => { + const plugin = await loadPlugin() + try { + const names = readdirSync(join(methodsDir, "tools")) + .filter((file) => file.endsWith(".json")) + .map((file) => (JSON.parse(readFileSync(join(methodsDir, "tools", file), "utf8")) as { name: string }).name) + assert.deepEqual([...plugin.tools.keys()].sort(), names.sort()) + for (const name of [ + "gaffer_hosts", + "gaffer_install_host", + "gaffer_context", + "gaffer_apply", + "gaffer_execute_code", + ]) + assert.ok(plugin.tools.has(name), name) + } finally { + await plugin.cleanup() + } + }) + + it("attaches the only free script at the first call, with the working method; listing asks no permission", async () => { + const plugin = await loadPlugin() + const host = await offered() + try { + await plugin.rpc("advertise", announcement(host.endpoint, host.scriptHandle)) + const listed = content(await plugin.tool("gaffer_hosts")) + assert.equal(listed.scripts.length, 1) + assert.equal(listed.attached_to, null) + assert.deepEqual(plugin.permissions, []) + const read = content(await plugin.tool("gaffer_context")) + assert.equal(read.revision, 3) + assert.equal(read.attached_to.label, "Gaffer: shot.gfr") + assert.deepEqual(plugin.methods, [{ sessionID: "ses_test", key: METHOD_KEY }]) + assert.deepEqual(plugin.permissions, []) + // Another conversation does not take the script this one uses: it is told who holds it. + await assert.rejects(plugin.tool("gaffer_context", {}, "ses_other"), /attached to another conversation/) + const shared = content(await plugin.tool("gaffer_hosts", { attach: listed.scripts[0].offerID }, "ses_other")) + assert.match(shared.shared, /Another conversation is attached to this Gaffer too/) + assert.deepEqual(plugin.asked, [ + { action: "gaffer_attach", resources: ["Gaffer: shot.gfr"], metadata: { label: "Gaffer: shot.gfr" } }, + ]) + } finally { + await plugin.cleanup() + await host.close() + } + }) + + // The panel attaches its conversation through the attachHost RPC: next to a CodeNomad conversation attached to the + // same script (gaffer_hosts), both stay attached, and either one's detach leaves the other (no lock, as Houdini). + it("the panel's conversation and another one are attached to the same script together", async () => { + const plugin = await loadPlugin() + const host = await offered() + try { + await plugin.rpc("advertise", announcement(host.endpoint, host.scriptHandle)) + const offerID = content(await plugin.tool("gaffer_hosts")).scripts[0].offerID + assert.equal(content(await plugin.tool("gaffer_hosts", { attach: offerID })).attached, "attached") + assert.deepEqual(await plugin.rpc("attachHost", { offerID, sessionID: "ses_other" }), { outcome: "attached" }) + const listed = content(await plugin.tool("gaffer_hosts", {}, "ses_other")) + assert.equal(listed.attached_to.offerID, offerID) + assert.equal(content(await plugin.tool("gaffer_context")).revision, 3) + assert.equal(content(await plugin.tool("gaffer_context", {}, "ses_other")).revision, 3) + // The panel closes: the other conversation keeps the script; and the other way round. + assert.deepEqual(await plugin.rpc("detachHost", { sessionID: "ses_other" }), { outcome: "detached" }) + assert.equal(content(await plugin.tool("gaffer_context")).attached_to.offerID, offerID) + assert.deepEqual(await plugin.rpc("attachHost", { offerID, sessionID: "ses_other" }), { outcome: "attached" }) + assert.equal(content(await plugin.tool("gaffer_hosts", { detach: true })).detached, "detached") + assert.equal(content(await plugin.tool("gaffer_context", {}, "ses_other")).attached_to.offerID, offerID) + } finally { + await plugin.cleanup() + await host.close() + } + }) + + it("attaching one of several scripts asks gaffer_attach; a refusal attaches nothing", async () => { + let deny = true + const plugin = await loadPlugin({ + answer: () => { + if (deny) throw new Error("Permission denied") + }, }) - const bound = (await plugin.rpc("bindingRead", { sessionID: "ses_test" })) as typeof before - assert.equal(bound.state, "attached") - assert.ok(bound.authority.revision > before.authority.revision) - assert.deepEqual(await plugin.rpc("bindingRead", { sessionID: "ses_other" }), { - state: "none", - authority: bound.authority, + const first = await offered() + const second = await offered() + try { + await plugin.rpc("advertise", announcement(first.endpoint, first.scriptHandle, "first", "Gaffer: a.gfr")) + await plugin.rpc("advertise", announcement(second.endpoint, second.scriptHandle, "second", "Gaffer: b.gfr")) + // Two free scripts: none is chosen for the conversation. + await assert.rejects(plugin.tool("gaffer_context"), /gaffer_hosts/) + const offers = content(await plugin.tool("gaffer_hosts")).scripts as Array<{ offerID: string; label: string }> + const b = offers.find((offer) => offer.label === "Gaffer: b.gfr")?.offerID as string + await assert.rejects(plugin.tool("gaffer_hosts", { attach: b }), /Permission denied/) + assert.deepEqual(plugin.permissions, ["gaffer_attach"]) + assert.equal(((await plugin.rpc("bindingRead", { sessionID: "ses_test" })) as { state: string }).state, "none") + deny = false + await plugin.tool("gaffer_hosts", { attach: b }) + assert.equal(content(await plugin.tool("gaffer_context")).attached_to.label, "Gaffer: b.gfr") + assert.deepEqual(plugin.permissions, ["gaffer_attach", "gaffer_attach"]) + } finally { + await plugin.cleanup() + await first.close() + await second.close() + } + }) + + it("two scripts with the same label are said, not left to look like one", async () => { + const plugin = await loadPlugin() + const first = await offered() + const second = await offered() + try { + await plugin.rpc("advertise", announcement(first.endpoint, first.scriptHandle, "first", "Gaffer: shot.gfr")) + assert.equal(content(await plugin.tool("gaffer_hosts")).same_label, undefined) + await plugin.rpc("advertise", announcement(second.endpoint, second.scriptHandle, "second", "Gaffer: shot.gfr")) + const listed = content(await plugin.tool("gaffer_hosts")) + assert.equal(listed.scripts.length, 2) + assert.match(listed.same_label, /Several Gaffers have the same file open \(Gaffer: shot\.gfr\)/) + } finally { + await plugin.cleanup() + await first.close() + await second.close() + } + }) + + it("says how to offer a script when none is", async () => { + const plugin = await loadPlugin() + try { + await assert.rejects( + plugin.tool("gaffer_context"), + /No Gaffer script is offered: start Gaffer 1\.6 or later with OpenCode Link installed \(gaffer_install_host\)/, + ) + } finally { + await plugin.cleanup() + } + }) + + it("execute_code asks gaffer_execute_code with the code for each run, reads a script file as is", async () => { + const plugin = await loadPlugin() + const dir = mkdtempSync(join(tmpdir(), "gaffer-script-")) + const runs: Array> = [] + const host = await offered({ + execute_code: (params) => { + runs.push(params) + return { status: "executed", result: { ok: 1 }, changed: true, saved: runs.length === 2 } + }, }) - plugin.sessions.set("ses_test", { id: "ses_test", location: { directory: join(tmpdir(), "foreign") } }) - assert.deepEqual(await plugin.rpc("bindingRead", { sessionID: "ses_test" }), { state: "unknown" }) - await assert.rejects(plugin.rpc("detachHost", { sessionID: "ses_test" }), /ownership/) - plugin.sessions.set("ses_test", { id: "ses_test", location: { ...plugin.location } }) - assert.deepEqual(await plugin.rpc("detachHost", { sessionID: "ses_test" }), { outcome: "detached" }) - assert.deepEqual(await plugin.rpc("advertise", announced), { outcome: "renewed" }) - assert.equal(((await plugin.rpc("bindingRead", { sessionID: "ses_test" })) as { state: string }).state, "none") - await plugin.rpc("attachHost", { offerID: offer.offerID, sessionID: "ses_test" }) - now += 15_001 - assert.deepEqual(await plugin.rpc("hosts"), { offers: [] }) - assert.equal(((await plugin.rpc("bindingRead", { sessionID: "ses_test" })) as { state: string }).state, "none") - assert.ok(plugin.events.every(({ data }) => JSON.stringify(data) === "{}")) - } finally { - await plugin.cleanup() - await host.close() - } -}) + try { + await plugin.rpc("advertise", announcement(host.endpoint, host.scriptHandle)) + const code = "result = {'ok': len(script.children())}" + const out = content(await plugin.tool("gaffer_execute_code", { code, read_only: true })) + assert.deepEqual([out.status, out.result], ["executed", { ok: 1 }]) + assert.deepEqual(plugin.asked, [{ action: "gaffer_execute_code", resources: ["*"], metadata: { code } }]) + assert.deepEqual(runs[0], { session_id: "ses_test", code, read_only: true, script_handle: host.scriptHandle }) + const file = join(dir, "build.py") + writeFileSync(file, 'import GafferScene\nscript.addChild(GafferScene.Sphere())\nprint("é")\n') + const saved = content(await plugin.tool("gaffer_execute_code", { path: file })) + assert.equal(runs[1]?.code, 'import GafferScene\nscript.addChild(GafferScene.Sphere())\nprint("é")\n') + assert.equal(runs[1]?.filename, file) + assert.equal(runs[1]?.read_only, undefined) + assert.equal(saved.check, verifySavedHint(true)) + assert.deepEqual(plugin.asked[1], { + action: "gaffer_execute_code", + resources: [file], + metadata: { path: file, code: runs[1]?.code }, + }) + await assert.rejects(plugin.tool("gaffer_execute_code", { code: "x", path: file }), /code or path, not both/) + assert.equal(runs.length, 2) + } finally { + await plugin.cleanup() + await host.close() + rmSync(dir, { recursive: true, force: true }) + } + }) -it("Gaffer permission denial and attachment changes prevent attaching", async () => { - const denied = await loadPlugin({ - answer: async () => { - throw new Error("Denied") - }, - }) - try { - await assert.rejects(denied.tool("gaffer_hosts"), /Denied/) - assert.deepEqual(denied.permissions, ["gaffer_attach"]) - assert.ok(denied.skills.has("gaffer-graph")) - } finally { - await denied.cleanup() - } - const moving = await loadPlugin({ - answer: async () => { - moving.sessions.delete("ses_test") - }, - }) - try { - await assert.rejects(moving.tool("gaffer_hosts"), /changed/) - } finally { - await moving.cleanup() - } -}) + it("execute_code is refused before any permission when the Gaffer does not list it", async () => { + const plugin = await loadPlugin() + const host = await offered() + try { + await plugin.rpc("advertise", announcement(host.endpoint, host.scriptHandle)) + await assert.rejects(plugin.tool("gaffer_execute_code", { code: "print(1)" }), /Code execution is not available/) + assert.deepEqual(plugin.permissions, []) + } finally { + await plugin.cleanup() + await host.close() + } + }) -it("WSL refuses host installation before writes; closed host remains unknown", async () => { - const plugin = await loadPlugin({ wsl: () => true }) - try { - const status = (await plugin.rpc("status")) as { host: { state: string } } - assert.equal(status.host.state, "unsupported") - assert.equal(((await plugin.rpc("installHost")) as { outcome: string }).outcome, "refused") - } finally { - await plugin.cleanup() - } - const supported = await loadPlugin() - try { - assert.equal(((await supported.rpc("status")) as { host: { state: string } }).host.state, "unknown") - } finally { - await supported.cleanup() - } -}) + it("apply asks gaffer_write with its operations; a render's previews are attachments", async () => { + const answers: string[] = [] + let deny = false + const plugin = await loadPlugin({ + answer: (action) => { + answers.push(action) + if (deny) throw new Error("Permission denied") + }, + }) + const preview = join(tmpdir(), "gaffer-preview.png") + const applied: Array> = [] + const host = await offered({ + apply: (params) => { + applied.push(params) + return { status: "applied", revision: 4, job: { outputs: [{ name: "beauty", preview_path: preview }] } } + }, + }) + try { + await plugin.rpc("advertise", announcement(host.endpoint, host.scriptHandle)) + const operations = [{ op: "render.start", target: "Render" }] + const out = (await plugin.tool("gaffer_apply", { if_revision: 3, command_id: "a", operations })) as unknown + const parts = (out as { content?: unknown }).content ?? out + assert.ok(Array.isArray(parts), JSON.stringify(out)) + assert.equal((parts as Array<{ type: string }>)[1]?.type, "file") + assert.ok(!(parts as Array<{ text?: string }>)[0]?.text?.includes(preview)) + assert.deepEqual(plugin.asked, [ + { action: "gaffer_write", resources: ["Gaffer: shot.gfr"], metadata: { operations } }, + ]) + assert.equal(applied[0]?.session_id, "ses_test") + deny = true + await assert.rejects(plugin.tool("gaffer_apply", { if_revision: 4, command_id: "b", operations }), /denied/) + assert.equal(applied.length, 1) + assert.deepEqual(answers, ["gaffer_write", "gaffer_write"]) + } finally { + await plugin.cleanup() + await host.close() + } + }) -it("host payload and installer: versioned offline archive, stable sidecar and unmanaged-file protection", async () => { - const scratch = mkdtempSync(join(tmpdir(), "gaffer-install-")) - const oldLocal = process.env.LOCALAPPDATA - const oldState = process.env.XDG_STATE_HOME - process.env.LOCALAPPDATA = join(scratch, "local") - process.env.XDG_STATE_HOME = join(scratch, "state") - try { - const marker = readPackage().opencodeIntegration - const payload = readFileSync(join(packageRoot, marker.host!.payload)) - assert.equal(createHash("sha256").update(payload).digest("hex"), marker.host!.sha256) - const entries = readStoredZip(payload) - assert.equal( - JSON.parse(entries.find((e) => e.name === "manifest.json")!.data.toString()).version, - readPackage().version, + it("a refusal reads the hint for its code with Gaffer's own text and data", async () => { + const plugin = await loadPlugin() + const host = await offered({ + release: () => { + throw { + code: "stale_revision", + message: "Read the script again (gaffer_context) and release with its revision.", + data: { current_revision: 7, reason: "revision_conflict" }, + } + }, + }) + try { + await plugin.rpc("advertise", announcement(host.endpoint, host.scriptHandle)) + await assert.rejects( + plugin.tool("gaffer_context", { release_quarantine: 2 }), + /^Error: The script changed since your last read: call gaffer_context and use its revision\. Read the script again \(gaffer_context\) and release with its revision\. \{"current_revision":7,"reason":"revision_conflict"\}$/, + ) + } finally { + await plugin.cleanup() + await host.close() + } + }) + + it("a call that outlasts its wait says Gaffer still runs it (status off the main thread)", async () => { + const temp = mkdtempSync(join(tmpdir(), "gaffer_temp-")) + const plugin = await loadPlugin() + const host = await offered( + { + execute_code: () => sleep(5_000).then(() => ({ status: "executed" })), + status: () => ({ busy: true, running: "execute_code", seconds: 1.2, queued: 0 }), + }, + { pid: process.pid, temp_dir: temp }, ) - const sidecar = entries.find((e) => e.name === "sidecar.js")!.data.toString() - assert.ok(!sidecar.includes("sidecar.ts")) - const startup = join(scratch, "startup", "gui") - assert.deepEqual(await installHost(marker, { hostStartupDir: startup }), { outcome: "installed" }) - assert.deepEqual(await installHost(marker, { hostStartupDir: startup }), { outcome: "unchanged" }) - const stable = join(stateRoot(), "hosts", marker.host!.sha256) - assert.ok(existsSync(join(stable, "sidecar.js"))) - const loader = join(startup, "neuralnomadsOpenCode.py") - assert.ok(!readFileSync(loader, "utf8").includes(packageRoot.replaceAll("\\", "/"))) - writeFileSync(loader, "# user file\n") - assert.equal((await installHost(marker, { hostStartupDir: startup })).outcome, "refused") - assert.equal(readFileSync(loader, "utf8"), "# user file\n") - assert.equal(readdirSync(join(stateRoot(), "hosts")).length, 1) - } finally { - if (oldLocal === undefined) delete process.env.LOCALAPPDATA - else process.env.LOCALAPPDATA = oldLocal - if (oldState === undefined) delete process.env.XDG_STATE_HOME - else process.env.XDG_STATE_HOME = oldState - rmSync(scratch, { recursive: true, force: true }) - } -}) + try { + await plugin.rpc("advertise", announcement(host.endpoint, host.scriptHandle)) + await assert.rejects( + plugin.tool("gaffer_execute_code", { code: "import time; time.sleep(5)", timeout_s: 1 }), + /Gaffer is still running execute_code \(1\.2 s so far\): it goes on in Gaffer, nothing is lost/, + ) + } finally { + await plugin.cleanup() + await host.close() + rmSync(temp, { recursive: true, force: true }) + } + }) -it("render feedback uses attachments and removes temporary image paths from text", () => { - const job = { outputs: [{ name: "beauty", preview_path: join(tmpdir(), "preview.png") }, { name: "pending" }] } - for (const result of [{ job }, { jobs: [job, { outputs: [] }] }, { job, jobs: [job] }]) { - const content = resultContent(result) - assert.ok(Array.isArray(content)) - assert.equal(content.length, "job" in result && "jobs" in result ? 3 : 2) - assert.equal(content[1]!.type, "file") - assert.ok(!(content[0] as { text: string }).text.includes("preview_path")) - assert.ok(!(content[0] as { text: string }).text.includes("preview.png")) - assert.ok(job.outputs[0]?.preview_path) - } + it("an earlier OpenCode installation in /gaffer is a notice and said by gaffer_hosts", async () => { + const paths = isolatedPaths() + const gui = join(paths.home, "gaffer", "startup", "gui") + mkdirSync(gui, { recursive: true }) + const plugin = await loadPlugin({ paths }) + try { + assert.deepEqual(((await plugin.rpc("status")) as { notices?: string[] }).notices ?? [], []) + assert.equal(content(await plugin.tool("gaffer_hosts")).legacy_install, undefined) + // A startup file of this integration's own is not an earlier installation. + writeFileSync(join(gui, "openCode.py"), `${HEADER}import runpy\n`) + assert.deepEqual(((await plugin.rpc("status")) as { notices?: string[] }).notices ?? [], []) + writeFileSync(join(gui, "openCode.py"), "import GafferOpenCode\n") + mkdirSync(join(paths.home, "gaffer", "OpenCode")) + assert.deepEqual(((await plugin.rpc("status")) as { notices?: string[] }).notices, ["legacy_install"]) + const listed = content(await plugin.tool("gaffer_hosts")).legacy_install + assert.deepEqual(listed.found, [ + { path: join(gui, "openCode.py"), kind: "startup file" }, + { path: join(paths.home, "gaffer", "OpenCode"), kind: "folder" }, + ]) + assert.match(listed.fix, /remove them, then restart Gaffer/) + rmSync(join(gui, "openCode.py")) + assert.deepEqual(((await plugin.rpc("status")) as { notices?: string[] }).notices, ["legacy_install"]) + } finally { + await plugin.cleanup() + rmSync(paths.home, { recursive: true, force: true }) + } + }) + + it("installs the managed loader and payload into the startup folder, idempotently; never an unmanaged file", async () => { + const root = mkdtempSync(join(tmpdir(), "gaffer-install-")) + const startup = join(root, "startup", "gui") + const state = join(root, "state") + const plugin = await loadPlugin({ options: { hostStartupDir: startup }, state }) + try { + const marker = readPackage().opencodeIntegration + const host = marker.host as NonNullable + const archive = readFileSync(join(packageRoot, host.payload)) + assert.equal(createHash("sha256").update(archive).digest("hex"), host.sha256) + const entries = readStoredZip(archive) + const manifest = entries.find((entry) => entry.name === "manifest.json")?.data.toString() ?? "{}" + assert.equal(JSON.parse(manifest).version, version) + assert.ok(entries.some((entry) => entry.name === "sidecar.cjs")) + assert.ok(entries.some((entry) => entry.name === "startup/gui/openCode.py")) + + assert.deepEqual(inspectHost(startup), { kind: "not-inspected" }) + assert.deepEqual(await plugin.rpc("installHost"), { + outcome: "installed", + detail: "Restart Gaffer: OpenCode Link offers its scripts to OpenCode when it starts.", + }) + const loader = join(startup, LOADER) + const source = readFileSync(loader, "utf8") + assert.ok(source.startsWith(HEADER)) + assert.ok(!source.includes(packageRoot.replaceAll("\\", "/")), "the loader starts the copy, not the package") + assert.ok(existsSync(join(state, "hosts", host.sha256, "startup", "gui", "openCode.py"))) + assert.deepEqual(inspectHost(startup), { kind: "found", version }) + assert.deepEqual(await plugin.rpc("installHost"), { outcome: "unchanged" }) + assert.deepEqual(readdirSync(join(state, "hosts")), [host.sha256]) + + writeFileSync(loader, "# the user's own startup file\n") + assert.deepEqual(inspectHost(startup), { kind: "not-inspected" }) + assert.equal(((await plugin.rpc("installHost")) as { outcome: string }).outcome, "refused") + assert.equal(readFileSync(loader, "utf8"), "# the user's own startup file\n") + } finally { + await plugin.cleanup() + rmSync(root, { recursive: true, force: true }) + } + }) + + it("gaffer_install_host asks its permission; a WSL daemon refuses before any write", async () => { + const root = mkdtempSync(join(tmpdir(), "gaffer-install-")) + const startup = join(root, "startup", "gui") + const plugin = await loadPlugin({ options: { hostStartupDir: startup }, state: join(root, "state") }) + const wsl = await loadPlugin({ options: { hostStartupDir: startup }, state: join(root, "never"), wsl: () => true }) + try { + assert.equal(content(await plugin.tool("gaffer_install_host")).outcome, "installed") + assert.deepEqual(plugin.permissions, ["gaffer_install_host"]) + rmSync(startup, { recursive: true, force: true }) + assert.equal(((await wsl.rpc("status")) as { host: { state: string } }).host.state, "unsupported") + assert.equal(((await wsl.rpc("installHost")) as { outcome: string }).outcome, "refused") + assert.equal(existsSync(join(root, "never")), false) + assert.equal(existsSync(startup), false) + } finally { + await plugin.cleanup() + await wsl.cleanup() + rmSync(root, { recursive: true, force: true }) + } + }) + + it("attaches only this Location's conversations: ownership, detach, renewal and expiry", async () => { + let now = 10_000 + const plugin = await loadPlugin({ now: () => now }) + const host = await offered() + try { + const announced = announcement(host.endpoint, host.scriptHandle) + assert.deepEqual(await plugin.rpc("advertise", { ...announced, host: "localhost" }), { outcome: "refused" }) + assert.deepEqual(await plugin.rpc("advertise", { ...announced, token: "0".repeat(64) }), { outcome: "refused" }) + // hello names another script than the announced one: refused. + assert.deepEqual(await plugin.rpc("advertise", { ...announced, script_handle: "gaffer:other" }), { + outcome: "refused", + }) + assert.deepEqual(await plugin.rpc("advertise", announced), { outcome: "accepted" }) + const offerID = plugin.registry.hosts().offers[0]?.offerID as string + assert.deepEqual(await plugin.rpc("attachHost", { offerID, sessionID: "ses_foreign" }), { outcome: "refused" }) + assert.deepEqual(await plugin.rpc("bindingRead", { sessionID: "ses_foreign" }), { state: "unknown" }) + assert.deepEqual(await plugin.rpc("attachHost", { offerID, sessionID: "ses_test" }), { outcome: "attached" }) + assert.equal( + ((await plugin.rpc("bindingRead", { sessionID: "ses_test" })) as { state: string }).state, + "attached", + ) + // A conversation moved to another Location is no longer this one's. + plugin.sessions.set("ses_test", { id: "ses_test", location: { directory: join(tmpdir(), "foreign") } }) + assert.deepEqual(await plugin.rpc("bindingRead", { sessionID: "ses_test" }), { state: "unknown" }) + await assert.rejects(plugin.rpc("detachHost", { sessionID: "ses_test" }), /ownership/) + plugin.sessions.set("ses_test", { id: "ses_test", location: { ...plugin.location } }) + assert.deepEqual(await plugin.rpc("detachHost", { sessionID: "ses_test" }), { outcome: "detached" }) + assert.deepEqual(await plugin.rpc("advertise", announced), { outcome: "renewed" }) + assert.equal(((await plugin.rpc("bindingRead", { sessionID: "ses_test" })) as { state: string }).state, "none") + now += 15_001 + assert.deepEqual(await plugin.rpc("hosts"), { offers: [] }) + assert.ok(plugin.events.every(({ data }) => JSON.stringify(data) === "{}")) + } finally { + await plugin.cleanup() + await host.close() + } + }) + + it("render feedback uses attachments and removes temporary image paths from the text", () => { + const job = { outputs: [{ name: "beauty", preview_path: join(tmpdir(), "preview.png") }, { name: "pending" }] } + for (const result of [{ job }, { jobs: [job, { outputs: [] }] }, { job, jobs: [job] }]) { + const parts = resultContent(result) + assert.ok(Array.isArray(parts)) + assert.equal(parts.length, "job" in result && "jobs" in result ? 3 : 2) + assert.equal(parts[1]?.type, "file") + assert.ok(!(parts[0] as { text: string }).text.includes("preview")) + assert.ok(job.outputs[0]?.preview_path, "the caller's result is left as it was") + } + assert.equal(resultContent({ status: "applied" }), '{"status":"applied"}') + }) }) diff --git a/integrations/gaffer/test/programs_gaffer.py b/integrations/gaffer/test/programs_gaffer.py new file mode 100644 index 0000000..eaa6a19 --- /dev/null +++ b/integrations/gaffer/test/programs_gaffer.py @@ -0,0 +1,477 @@ +"""The fixed programs under a real Gaffer (opt-in: gaffer.test.ts with GAFFER_TEST_BIN = Gaffer's launcher). + +Run with: python programs_gaffer.py [test names...], in an isolated environment (private HOME, +USERPROFILE, TEMP and startup paths: gaffer.test.ts sets them, never the user's). Builds scripts in a temp folder, +runs each program as the plugin sends it (PARAMS, and JOB for verify_saved and render, prepended; exec with the Gaffer +modules, script, application and result) and checks the results. Evidence (each program's result) is written to +OPENCODE_GAFFER_EVIDENCE as programs_gaffer.json when that variable names a folder. +""" + +import json +import os +import shutil +import sys +import tempfile +import time +import traceback + +try: + import Gaffer + import GafferDispatch + import GafferImage + import GafferScene + import IECore + import IECoreScene + import imath +except ImportError: + print("SKIP: run with Gaffer's launcher: gaffer python programs_gaffer.py") + raise SystemExit(0) + +HERE = os.path.dirname(os.path.abspath(sys.argv[0] if sys.argv and sys.argv[0].endswith(".py") else __file__)) +PROGRAMS = os.path.join(HERE, "..", "plugin", "programs") +WORK = tempfile.mkdtemp(prefix="oc_gaffer_programs_") +EVIDENCE = {} +RESULTS = [] +SCRIPT = None + + +def source(name): + with open(os.path.join(PROGRAMS, name), encoding="utf-8") as fh: + return fh.read() + + +def program(name, params, job=None): + """As the plugin sends it: PARAMS (and JOB) prepended, run with exec in the installed part's namespace.""" + text = "import json\nPARAMS = json.loads(%s)\n" % json.dumps(json.dumps(params)) + if job is not None: + text += "JOB = %s\n" % json.dumps(source(job)) + text += source(name) + namespace = {"Gaffer": Gaffer, "GafferScene": GafferScene, "GafferImage": GafferImage, "GafferDispatch": GafferDispatch, + "IECore": IECore, "imath": imath, "script": SCRIPT, "application": None, "__name__": "__opencode__", "result": {}} + exec(compile(text, "", "exec"), namespace) # noqa: S102 + return json.loads(json.dumps(namespace.get("result", {}), default=repr)) + + +def failing(name, params, job=None): + try: + program(name, params, job) + except Exception as exc: # noqa: BLE001 + return "%s: %s" % (type(exc).__name__, exc) + raise AssertionError("%s with %s did not fail" % (name, params)) + + +def verify_saved(params, timeout=300): + """gaffer_verify saved as the plugin runs it: the program starts the background gaffer python and returns at + once; its result is followed from outside (saved-verify.ts). Returns (result, steps of its log, seconds to start).""" + started_at = time.time() + start = program("verify_saved.py", params, job="verify.py")["started"] + start_s = time.time() - started_at + deadline = time.time() + timeout + while time.time() < deadline and not os.path.isfile(start["result"]): + time.sleep(0.2) + with open(start["log"], encoding="utf-8", errors="replace") as fh: + log = fh.read() + steps = [json.loads(line[14:]) for line in log.splitlines() if line.startswith("OPENCODE_STEP ")] + if not os.path.isfile(start["result"]): + raise AssertionError("no result after %d s; log:\n%s" % (timeout, log[-2000:])) + with open(start["result"], encoding="utf-8") as fh: + result = json.load(fh) + if "error" in result: + raise AssertionError(result["error"]) + result["read"] = {"file": start["file"], "bytes": start["bytes"], "seconds": round(time.time() - started_at, 1)} + shutil.rmtree(start["work"], ignore_errors=True) + return result, steps, start_s + + +def wait_job(started, timeout=300): + """Follow the log as the plugin's task tracker does: markers until DONE or FAILED.""" + deadline = time.time() + timeout + marks = [] + while time.time() < deadline: + with open(started["log"], encoding="utf-8", errors="replace") as fh: + lines = fh.read().splitlines() + marks = [line for line in lines if line.startswith("OPENCODE_")] + if any(line.startswith(("OPENCODE_DONE ", "OPENCODE_FAILED ")) for line in marks): + break + time.sleep(0.5) + return [(line.split(" ", 1)[0][9:], json.loads(line.split(" ", 1)[1])) for line in marks] + + +def check(condition, message): + if not condition: + raise AssertionError(message) + + +TESTS = [] + + +def test(fn): + TESTS.append(fn) + return fn + + +def new_script(): + global SCRIPT + SCRIPT = Gaffer.ScriptNode() + return SCRIPT + + +def scene_script(): + """sphere (radius 2, in set 'hero') and cube -> group; camera; an expression driving cube size by frame; a + sphere 'bad' whose radius expression divides by zero.""" + s = new_script() + s["sphere"] = GafferScene.Sphere() + s["sphere"]["radius"].setValue(2) + s["sphere"]["sets"].setValue("hero") + s["cube"] = GafferScene.Cube() + s["camera"] = GafferScene.Camera() + s["camera"]["transform"]["translate"].setValue(imath.V3f(0, 0, 8)) + s["group"] = GafferScene.Group() + for i, name in enumerate(("sphere", "cube", "camera")): + s["group"]["in"][i].setInput(s[name]["out"]) + s["attrs"] = GafferScene.StandardAttributes() + s["attrs"]["in"].setInput(s["group"]["out"]) + s["attrs"]["attributes"]["scene:visible"]["enabled"].setValue(True) + s["filter"] = GafferScene.PathFilter() + s["filter"]["paths"].setValue(IECore.StringVectorData(["/group/sphere"])) + s["attrs"]["filter"].setInput(s["filter"]["out"]) + s["cubeSize"] = Gaffer.Expression() + s["cubeSize"].setExpression('parent["cube"]["dimensions"]["x"] = context.getFrame()', "python") + s["bad"] = GafferScene.Sphere() + s["badRadius"] = Gaffer.Expression() + s["badRadius"].setExpression('parent["bad"]["radius"] = 1.0 / 0', "python") + return s + + +def fingerprint(): + return SCRIPT.serialise(), SCRIPT.context().getFrame(), SCRIPT["fileName"].getValue() + + +# --- inspect --------------------------------------------------------------------------------------------------- +@test +def inspect_type(): + new_script() + r = program("inspect.py", {"type": "GafferScene.Sphere"}) + EVIDENCE["inspect_type"] = r + t = r["type"] + check(r["gaffer"]["version"] == Gaffer.About.versionString() and r["gaffer"]["descriptions"], r["gaffer"]) + check(t["type"] == "GafferScene.Sphere" and t["description"].startswith("Produces scenes containing a sphere"), + {k: v for k, v in t.items() if k != "plugs"}) + check(t["doc"].endswith("Reference/NodeReference/GafferScene/Sphere.html"), t["doc"]) + radius = next(p for p in t["plugs"] if p["name"] == "radius") + check(radius["type"] == "FloatPlug" and radius["default"] == 1.0 and radius["description"] == "Radius of the sphere.", radius) + translate = next(p for p in t["plugs"] if p["name"] == "transform.translate") + check(translate["type"] == "V3fPlug" and translate["default"] == [0.0, 0.0, 0.0], translate) + kind = next(p for p in t["plugs"] if p["name"] == "type") + check(kind.get("presets"), kind) + r = program("inspect.py", {"type": "Spheer"}) + check("error" in r["type"] and "GafferScene.Sphere" in r["type"]["closest"], r["type"]) + r = program("inspect.py", {"type": "Sphere", "match": "radius"}) + check([p["name"] for p in r["type"]["plugs"]] == ["radius"], r["type"]["plugs"]) + r = program("inspect.py", {"type": "GafferImage.ImageWriter"}) + check(r["type"].get("task") and any(p["name"] == "fileName" for p in r["type"]["plugs"]), r["type"]) + + +@test +def inspect_node(): + scene_script() + r = program("inspect.py", {"node": "attrs"}) + EVIDENCE["inspect_node_scene"] = r + n = r["node"] + check(n["type"] == "GafferScene.StandardAttributes" and n["description"], n) + check(any(p["name"] == "in" and p["from"] == "group.out" for p in n["plugs_changed"]), n["plugs_changed"]) + scene = n["scene"]["out"] + paths = [entry["path"] for entry in scene["locations"]] + check(paths[:4] == ["/group", "/group/sphere", "/group/cube", "/group/camera"], paths) + sphere = next(entry for entry in scene["locations"] if entry["path"] == "/group/sphere") + check(sphere["object"] == "MeshPrimitive" and "scene:visible" in sphere["attributes"], sphere) + check(scene["sets"].get("hero") == 1 and "__cameras" in scene["sets"], scene["sets"]) + check(scene["bound"]["min"][0] == -2.0, scene["bound"]) + r = program("inspect.py", {"node": "sphere", "match": "radius"}) + check(next(p for p in r["node"]["plugs_changed"] if p["name"] == "radius")["value"] == 2.0, r["node"]) + r = program("inspect.py", {"node": "bad"}) + EVIDENCE["inspect_node_error"] = r["node"].get("errors") + check(r["node"].get("errors") and "ZeroDivisionError" in r["node"]["errors"][0]["error"], r["node"]) + r = program("inspect.py", {"node": "spheer"}) + check("sphere" in r["node"]["closest"], r["node"]) + r = program("inspect.py", {"node": "/"}) + check(r["node"]["path"] == "/" and any(c["name"] == "group" for c in r["node"]["children"]), r["node"]) + s = SCRIPT + s["checker"] = GafferImage.Checkerboard() + s["checker"]["format"].setValue(GafferImage.Format(64, 32)) + r = program("inspect.py", {"node": "checker"}) + check(r["node"]["image"]["out"]["format"] == [64, 32] and "R" in r["node"]["image"]["out"]["channels"], r["node"].get("image")) + + +@test +def inspect_gaffer_and_search(): + new_script() + r = program("inspect.py", {"gaffer": "Gaffer.GraphComponent.setName"}) + check("setName(" in r["gaffer"]["doc"] and r["gaffer"]["kind"] == "callable", r["gaffer"]) + r = program("inspect.py", {"gaffer": "Gaffer.Node"}) + check(r["gaffer"]["kind"] == "class" and "GraphComponent" in str(r["gaffer"]["bases"]), r["gaffer"]) + r = program("inspect.py", {"gaffer": "Gaffer.Nod"}) + check("Gaffer.Node" in r["gaffer"]["closest"], r["gaffer"]) + started = time.time() + r = program("inspect.py", {"search": "sphere", "limit": 15}) + first = time.time() - started + EVIDENCE["inspect_search"] = r + paths = [h["path"] for h in r["search"]["results"]] + check("GafferScene.Sphere" in paths[:3], paths) + started = time.time() + r = program("inspect.py", {"search": "radius"}) + check(any(h["kind"] == "plug" and h["path"] == "radius" and ("GafferScene.Sphere" in h["on"] or h.get("on_count", 0) > 8) + for h in r["search"]["results"]), + r["search"]["results"][:5]) + EVIDENCE["inspect_search_seconds"] = {"first": round(first, 2), "cached": round(time.time() - started, 2)} + message = failing("inspect.py", {}) + check("give search" in message, message) + + +# --- verify ---------------------------------------------------------------------------------------------------- +CRITERIA = [ + {"expr": "child_names('group', '/group')", "op": "==", "value": ["sphere", "cube", "camera"], "label": "group children"}, + {"expr": "bound('sphere', '/sphere').size", "op": "approx", "value": [4, 4, 4], "tol": 1e-5}, + {"expr": "exists('attrs', '/group/sphere')", "op": "==", "value": True}, + {"expr": "attributes('attrs', '/group/sphere')['scene:visible']", "op": "==", "value": True}, + {"expr": "set_members('attrs', 'hero')", "op": "==", "value": ["/group/sphere"]}, + {"expr": "plug('sphere.radius')", "op": "==", "value": 2}, + {"expr": "plug('camera.transform.translate')", "op": "==", "value": [0, 0, 8]}, + {"expr": "translate('group', '/group/camera')", "op": "approx", "value": [0, 0, 8], "tol": 1e-5}, + {"expr": "world_bound('group', '/group/sphere').size", "op": "approx", "value": [4, 4, 4], "tol": 1e-5}, + {"expr": "bound('cube', '/cube').size[0]", "op": "approx", "value": 10, "frame": 10, "label": "cube follows the frame at 10"}, + {"expr": "object_type('group', '/group/cube')", "op": "==", "value": "MeshPrimitive"}, + {"expr": "len(script.children(Gaffer.Node))", "op": ">=", "value": 8}, + {"expr": "bound('sphere', '/sphere').size[0]", "op": ">", "value": 100, "label": "deliberately failing"}, + {"expr": "bound('nothing', '/')", "op": "==", "value": 0, "label": "missing node"}, + {"expr": "bound('bad', '/sphere')", "op": "==", "value": 0, "label": "compute error"}, +] + + +@test +def verify_criteria_and_errors(): + scene_script() + before = fingerprint() + r = program("verify.py", {"criteria": CRITERIA}) + EVIDENCE["verify"] = r + check(fingerprint() == before, "the script or frame changed") + by = {c["label"]: c for c in r["criteria"]} + check(r["passed"] == len(CRITERIA) - 3 and r["failed"] == 3, + [(c["label"], c["passed"], c.get("measured"), c.get("error")) for c in r["criteria"]]) + check(by["deliberately failing"]["measured"] == 4.0 and "no node" in by["missing node"]["error"], by) + check("ZeroDivisionError" in by["compute error"]["error"], by["compute error"]) + check(by["cube follows the frame at 10"]["passed"] and SCRIPT.context().getFrame() == 1, by["cube follows the frame at 10"]) + bad = [e for e in r["errors"] if e["node"] == "bad"] + check(bad and not bad[0]["arises_here"] and "ZeroDivisionError" in bad[0]["errors"][0]["error"], r["errors"]) + check(bad[0]["errors"][0]["arose_in"] == "badRadius.__execute", bad) + scan = r["facts"]["error_scan"] + check(scan["errors_arise_in"] == ["badRadius"] and scan["nodes_with_errors"] == 1 and scan["nodes_read"] >= 9, scan) + check([e["node"] for e in r["errors"]] == ["bad"], r["errors"]) + for expr in ("script.save()", "node('sphere')['radius'].setValue(3)", "script.serialiseToFile('x.gfr')", + "node('sphere').addChild(Gaffer.Node())", "application"): + message = failing("verify.py", {"criteria": [{"expr": expr, "op": "==", "value": 1}]}) + check("not a read" in message or "names" in message, message) + r = program("verify.py", {"roots": ["sphere"], "errors": True}) + check(r["facts"]["error_scan"]["nodes_read"] == 1 and not r["errors"], r["facts"]) + + +@test +def verify_render(): + scene_script() + before = fingerprint() + started = time.time() + r = program("verify.py", {"errors": False, "render": {"size": 96, "scene": "group"}}) + EVIDENCE["verify_render"] = dict(r, image_base64=len(r.get("image_base64") or "")) + EVIDENCE["verify_render_seconds"] = round(time.time() - started, 1) + check(fingerprint() == before, "the script changed") + render = r["facts"]["render"] + check(r.get("image_base64") and render["image"]["size"] == [96, 72] and not render["image"]["black"], render) + check(render["camera"] == "/group/camera", render) + message = failing("verify.py", {"errors": False, "render": {"size": 96}}) + check("give render.scene" in message, message) + + +# --- save and verify saved ------------------------------------------------------------------------------------- +@test +def save_and_verify_saved(): + scene_script() + message = failing("save.py", {}) + check("never saved" in message, message) + first = os.path.join(WORK, "shot.gfr") + r = program("save.py", {"path": first}) + EVIDENCE["save"] = r + check(os.path.normcase(r["open_file"]) == os.path.normcase(first) and r["bytes"] > 1000 and not r["unsaved_changes"], r) + copy = os.path.join(WORK, "copies", "shot_copy.gfr") + r = program("save.py", {"path": copy, "copy": True}) + EVIDENCE["save_copy"] = r + check(os.path.isfile(copy) and os.path.normcase(r["open_file"]) == os.path.normcase(first) and r["copy"], r) + message = failing("save.py", {"path": copy, "copy": True}) + check("overwrite" in message, message) + message = failing("save.py", {"path": os.path.join(WORK, "shot.hip")}) + check(".gfr" in message, message) + reloaded = Gaffer.ScriptNode() + reloaded["fileName"].setValue(copy) + reloaded.load() + check(reloaded["sphere"]["radius"].getValue() == 2 and "badRadius" in reloaded, reloaded.keys()) + saved_nodes = len(SCRIPT.children(Gaffer.Node)) + time.sleep(0.05) + r = program("save.py", {}) + check(os.path.normcase(r["saved"]) == os.path.normcase(first), r) + SCRIPT["unsaved"] = GafferScene.Plane() # in the open script only + criteria = [{"expr": "len(script.children(Gaffer.Node))", "op": "==", "value": saved_nodes, "label": "nodes in the file"}, + {"expr": "plug('sphere.radius')", "op": "==", "value": 2}] + live = program("verify.py", {"criteria": criteria, "errors": False}) + check(live["failed"] == 1, live["criteria"]) + r, steps, start_s = verify_saved({"saved": True, "criteria": criteria}) + EVIDENCE["verify_saved"] = r + EVIDENCE["verify_saved_steps"] = steps + check(r["all_passed"] and r["read"]["bytes"] > 1000, r) + check(any(e["node"] == "bad" for e in r["errors"]), r["errors"]) + # Started, not waited for, on the main thread; its log says each step. + check(start_s < 3, start_s) + check([s["step"] for s in steps][:4] == ["load", "loaded", "criterion", "criterion"] and steps[-1]["step"] == "done", steps) + check(steps[2]["label"] == "nodes in the file" and steps[2]["index"] == 1 and steps[2]["of"] == 2, steps[2]) + check("errors" in [s["step"] for s in steps] and {"step": "cook", "node": "bad"} in [{k: v for k, v in s.items() if k != "at_s"} for s in steps], steps) + r2, _, _ = verify_saved({"saved": copy, "criteria": criteria[1:], "errors": False, "render": {"size": 64, "scene": "group"}}) + EVIDENCE["verify_saved_render"] = dict(r2, image_base64=len(r2.get("image_base64") or "")) + check(r2["all_passed"] and r2.get("image_base64") and "this background" in r2["facts"]["render"]["by"], r2) + EVIDENCE["verify_saved_seconds"] = {"criteria_and_errors": r["read"]["seconds"], "with_render": r2["read"]["seconds"]} + message = failing("verify_saved.py", {"saved": os.path.join(WORK, "missing.gfr")}, job="verify.py") + check("no file" in message, message) + + +@test +def verify_saved_reports_load_errors(): + s = new_script() + s["sphere"] = GafferScene.Sphere() + path = os.path.join(WORK, "broken.gfr") + s["fileName"].setValue(path.replace("\\", "/")) + s.save() + with open(path, encoding="utf-8") as fh: + text = fh.read() + with open(path, "w", encoding="utf-8") as fh: + fh.write(text + "\n__children[\"nothing\"][\"radius\"].setValue( 3 )\n") + r, _, _ = verify_saved({"saved": True, "criteria": [{"expr": "plug('sphere.radius')", "op": "==", "value": 1}], "errors": False}) + EVIDENCE["verify_saved_load_warnings"] = r.get("load_warnings") + check(r["all_passed"] and r.get("load_warnings"), r) + + +# --- render ---------------------------------------------------------------------------------------------------- +def image_script(): + s = new_script() + s["checker"] = GafferImage.Checkerboard() + s["checker"]["format"].setValue(GafferImage.Format(48, 32)) + s["writer"] = GafferImage.ImageWriter() + s["writer"]["in"].setInput(s["checker"]["out"]) + s["writer"]["fileName"].setValue(os.path.join(WORK, "own", "own.####.png").replace("\\", "/")) + return s + + +@test +def verify_errors_skip_private_internals(): + # Trial on a windowed Gaffer: an ImageWriter's private __contextQuery (a compound output without a value of its + # own) read as an error of a sound script. + image_script() + r = program("verify.py", {"criteria": []}) + EVIDENCE["verify_image_writer"] = r["facts"]["error_scan"] + check(r["errors"] == [] and r["facts"]["error_scan"]["nodes_with_errors"] == 0, r["errors"]) + + +@test +def render_image_writer_frames(): + image_script() + output = os.path.join(WORK, "out", "check_####.png") + started = time.time() + r = program("render.py", {"node": "writer", "output": output, "frames": [1, 3]}, job="render_job.py") + returned = time.time() - started + EVIDENCE["render_writer_start"] = r + check(SCRIPT["fileName"].getValue() == "", "the open file name changed") + check(r["kind"] == "render" and len(r["targets"]) == 3 and r["targets"][0].endswith("check_0001.png") and r["start"] == 1 and r["end"] == 3, r) + check(SCRIPT["writer"]["fileName"].getValue().endswith("own.####.png"), "the open script's output changed") + marks = wait_job(r) + EVIDENCE["render_writer_marks"] = marks + EVIDENCE["render_writer_seconds"] = {"returned": round(returned, 2), "finished": round(time.time() - started, 1)} + kinds = [k for k, _ in marks] + check(kinds and kinds[0] == "RENDER" and kinds[-1] == "DONE" and isinstance(marks[0][1]["pid"], int), marks) + check([v["done"] for k, v in marks if k == "PROGRESS"][-1] == 3, marks) + check(all(os.path.isfile(t) for t in r["targets"]) and marks[-1][1]["image_size"] == [48, 32], marks[-1]) + check(not os.path.isdir(os.path.join(WORK, "own")), "the node's own output was written") + message = failing("render.py", {"node": "writer", "output": output, "frames": [1, 3]}, job="render_job.py") + check("overwrite" in message, message) + message = failing("render.py", {"node": "checker"}, job="render_job.py") + check("not a task node" in message, message) + message = failing("render.py", {"node": "writer", "output": output.replace("####", "x"), "frames": [1, 2]}, job="render_job.py") + check("frame number" in message, message) + r = program("render.py", {"node": "writer"}, job="render_job.py") + check(r["targets"][0].endswith("own.0001.png"), r["targets"]) + check(wait_job(r)[-1][0] == "DONE", "the node's own output") + SCRIPT["missing"] = GafferImage.ImageReader() + SCRIPT["missing"]["fileName"].setValue(os.path.join(WORK, "nowhere.exr").replace("\\", "/")) + SCRIPT["writer"]["in"].setInput(SCRIPT["missing"]["out"]) + r = program("render.py", {"node": "writer", "output": os.path.join(WORK, "failed", "f.png")}, job="render_job.py") + marks = wait_job(r) + EVIDENCE["render_failed_marks"] = marks + check(marks[-1][0] == "FAILED" and "nowhere.exr" in marks[-1][1], marks) + + +@test +def render_cycles_size(): + s = scene_script() + del s["bad"], s["badRadius"] + s["outputs"] = GafferScene.Outputs() + s["outputs"]["in"].setInput(s["attrs"]["out"]) + s["outputs"].addOutput("beauty", IECoreScene.Output(os.path.join(WORK, "own_beauty.####.exr").replace("\\", "/"), "exr", "rgba", {})) + s["options"] = GafferScene.StandardOptions() + s["options"]["in"].setInput(s["outputs"]["out"]) + s["options"]["options"]["render:camera"]["enabled"].setValue(True) + s["options"]["options"]["render:camera"]["value"].setValue("/group/camera") + s["render"] = GafferScene.Render() + s["render"]["in"].setInput(s["options"]["out"]) + s["render"]["renderer"].setValue("Cycles") + output = os.path.join(WORK, "cycles", "beauty.####.exr") + r = program("render.py", {"node": "render", "output": output, "frames": 2, "size": [64, 48]}, job="render_job.py") + EVIDENCE["render_cycles_start"] = r + check(r["facts"]["renderer"] == "Cycles" and r["targets"][0].endswith("beauty.0002.exr"), r) + marks = wait_job(r) + EVIDENCE["render_cycles_marks"] = marks + check(marks and marks[-1][0] == "DONE" and marks[-1][1]["image_size"] == [64, 48], marks) + check(not os.path.exists(os.path.join(WORK, "own_beauty.0002.exr")), "the script's own output was rendered") + message = failing("render.py", {"node": "writer" if "writer" in s else "outputs"}, job="render_job.py") + check("not a task node" in message, message) + message = failing("render.py", {"node": "render", "output": output, "frames": 2}, job="render_job.py") + check("overwrite" in message, message) + + +def main(): + names = set(sys.argv[1:]) + started = time.time() + # Relative paths a program might write go to the run's own folder. + os.chdir(WORK) + for fn in TESTS: + if names and fn.__name__ not in names: + continue + t = time.time() + try: + fn() + RESULTS.append((fn.__name__, "ok", round(time.time() - t, 1), "")) + except Exception: # noqa: BLE001 + RESULTS.append((fn.__name__, "FAIL", round(time.time() - t, 1), traceback.format_exc()[-3000:])) + for name, status, seconds, detail in RESULTS: + print("%-34s %-4s %6.1fs %s" % (name, status, seconds, ("\n" + detail) if detail else "")) + folder = os.environ.get("OPENCODE_GAFFER_EVIDENCE") + if folder: + os.makedirs(folder, exist_ok=True) + with open(os.path.join(folder, "programs_gaffer.json"), "w", encoding="utf-8") as fh: + json.dump({"gaffer": Gaffer.About.versionString(), "results": [r[:3] for r in RESULTS], "evidence": EVIDENCE}, + fh, indent=1, default=repr) + failed = [r for r in RESULTS if r[1] != "ok"] + print("%d passed, %d failed in %.1f s" % (len(RESULTS) - len(failed), len(failed), time.time() - started)) + os.chdir(tempfile.gettempdir()) + shutil.rmtree(WORK, ignore_errors=True) + return 1 if failed else 0 + + +if __name__ in ("__main__", "__opencode_test__"): + code = main() + sys.stdout.flush() + os._exit(code) diff --git a/integrations/gaffer/test/python/real_host.py b/integrations/gaffer/test/python/real_host.py index 51ccdcc..5cbed6f 100644 --- a/integrations/gaffer/test/python/real_host.py +++ b/integrations/gaffer/test/python/real_host.py @@ -1,15 +1,26 @@ -"""Run with `gaffer python real_host.py` in an isolated environment.""" +"""Run with `gaffer python real_host.py` in an isolated environment (real-host.test.ts sets it up). + +The installed part in a real Gaffer, its interface's event loop running offscreen: the typed apply, the panel's +construction, the wire, hello and status off the main thread, execute_code (undo step, rollback, hints, quarantine and +its release), the busy answer while the main thread runs another request, and each script's offer and label. +""" import hashlib import json import pathlib -import sys -import types import socket +import sys import threading +import time +import types + sys.path.insert(0, str(pathlib.Path(sys.argv[0]).resolve().parents[2] / "host" / "python")) import Gaffer +import GafferScene from GafferOpenCode.Host import HostError, ScriptState +HOST_VERSION = json.loads((pathlib.Path(sys.argv[0]).resolve().parents[2] / "host" / "manifest.json").read_text())["version"] + +# --- the typed apply on a ScriptState -------------------------------------------------------------------------- script = Gaffer.ScriptNode("PrivateSmoke") source = Gaffer.Node("source") source.addChild(Gaffer.IntPlug("value", defaultValue=1)) @@ -19,7 +30,7 @@ hello = state.hello() canonical = json.dumps(hello["capabilities"], sort_keys=True, separators=(",", ":")) assert hello["capability_hash"] == hashlib.sha256(canonical.encode()).hexdigest() - assert hello["host_version"] == "0.2.0" + assert hello["host_version"] == HOST_VERSION context = state.context({"limit": 10, "include": ["values"]}) card = next(card for card in context["entities"] if card["name"] == "source") intent = {"session_id": "private-smoke", "script_handle": state.handle, @@ -59,59 +70,127 @@ finally: state.close() -# Construct the retained panel under offscreen Qt. Opening it does not launch the -# sidecar or advertise: the explicit Connect button is the consent boundary. +# --- the installed part as Gaffer's interface starts it, and the panel ------------------------------------------ +# Opening the panel does not start its sidecar nor touch the offers: Connect is the consent boundary. import GafferOpenCode from GafferOpenCodeUI.OpenCodeEditor import OpenCodeEditor root = Gaffer.ApplicationRoot("PrivatePanel") root["scripts"].addChild(script) -GafferOpenCode.initialise(types.SimpleNamespace(root=lambda: root)) +second = Gaffer.ScriptNode() +root["scripts"].addChild(second) +third = Gaffer.ScriptNode() +root["scripts"].addChild(third) +GafferOpenCode.initialise(types.SimpleNamespace(root=lambda: root), advertise=False) +host = GafferOpenCode.currentHost() editor = OpenCodeEditor(script) assert editor._OpenCodeEditor__channel is None assert editor._OpenCodeEditor__client is None assert editor._OpenCodeEditor__connectButton.getText() == "Connect" -assert GafferOpenCode.scriptHandle(script) not in editor._OpenCodeEditor__connectParams["script_key"] print("GAFFER_PANEL_OK") +# Each script is its own offer, labelled with its file (numbered when two scripts share it), under its own key. +offers = {handle: (label, key) for handle, label, key in host.offers()} +assert len(offers) == 3, offers +assert len({key for _, key in offers.values()}) == 3, offers +labels = sorted(label for label, _ in offers.values()) +assert labels[0] == "Gaffer: PrivateSmoke" and len(set(labels)) == 3, labels +# The label follows the fileName plug (nothing is written); two scripts with the same file are numbered. +second["fileName"].setValue("D:/shots/ex03.gfr") +third["fileName"].setValue("D:/shots/ex03.gfr") +assert GafferOpenCode.offerInfo(second)["label"] == "Gaffer: ex03.gfr", GafferOpenCode.offerInfo(second) +assert GafferOpenCode.offerInfo(third)["label"] == "Gaffer: ex03.gfr (2)", GafferOpenCode.offerInfo(third) +print("GAFFER_OFFERS_OK " + json.dumps(sorted(label for _, label, _ in host.offers()))) + import GafferUI info = GafferOpenCode.hostInfo() handle = GafferOpenCode.scriptHandle(script) assert info["host"] == "127.0.0.1" assert len(bytes.fromhex(info["token"])) == 32 loop = GafferUI.EventLoop.mainEventLoop() -wire = [] + + +def ask(method, params, token=None, timeout=60): + with socket.create_connection((info["host"], info["port"]), timeout=timeout) as client: + request = {"id": "probe", "token": token or info["token"], "method": method, "params": params} + client.sendall((json.dumps(request) + "\n").encode()) + return json.loads(client.makefile("rb").readline()) + + +checks = [] +evidence = {} + + def probe(): try: - for valid in (False, True): - with socket.create_connection((info["host"], info["port"]), timeout=5) as client: - request = {"id": "real-probe", "token": info["token"] if valid else "0" * 64, - "method": "hello", "params": {"script_handle": handle}} - client.sendall((json.dumps(request) + "\n").encode()) - response = json.loads(client.makefile("rb").readline()) - assert response["ok"] == valid - if valid: - assert response["result"]["script_handle"] == handle - assert response["result"]["host_version"] == "0.2.0" - with socket.create_connection((info["host"], info["port"]), timeout=5) as client: - client.sendall((json.dumps({"id": "bad-handle", "token": info["token"], "method": "hello", "params": {"script_handle": "gone"}}) + "\n").encode()) - response = json.loads(client.makefile("rb").readline()) - assert response["error"]["code"] == "stale_handle" + assert ask("hello", {"script_handle": handle}, token="0" * 64)["ok"] is False + hello = ask("hello", {"script_handle": handle})["result"] + assert hello["script_handle"] == handle and hello["host_version"] == HOST_VERSION + assert hello["process"]["pid"] > 0 and hello["process"]["temp_dir"] + assert {"execute_code", "release", "status"} <= set(hello["capabilities"]["methods"]) + assert ask("hello", {})["result"]["process"] == hello["process"] + assert ask("hello", {"script_handle": "gone"})["error"]["code"] == "stale_handle" # A refusal keeps the host's message, its data and its own code (data.reason). - with socket.create_connection((info["host"], info["port"]), timeout=5) as client: - request = {"id": "newer", "token": info["token"], "method": "context", - "params": {"script_handle": handle, "session_id": "private-smoke", "if_revision": 10 ** 6}} - client.sendall((json.dumps(request) + "\n").encode()) - error = json.loads(client.makefile("rb").readline())["error"] - assert error["code"] == "stale_revision", error - assert error["message"] == "Requested revision is newer than the Gaffer graph", error - assert error["data"]["reason"] == "revision_conflict", error - assert isinstance(error["data"]["current_revision"], int), error - wire.append("ok") + error = ask("context", {"script_handle": handle, "session_id": "private-smoke", "if_revision": 10 ** 6})["error"] + assert error["code"] == "stale_revision" and error["data"]["reason"] == "revision_conflict", error + checks.append("wire") + + # execute_code: one undo step; a failure after a change is undone, with Gaffer's hint. + ok = ask("execute_code", {"script_handle": handle, "code": "script['box'] = GafferScene.Cube()\nresult = {'n': len(script.children(Gaffer.Node))}"})["result"] + assert ok["status"] == "executed" and ok["changed"] and ok["undo_step"] == "OpenCode: code", ok + bad = ask("execute_code", {"script_handle": handle, "code": "script['plane'] = GafferScene.Plane()\nscript['box']['dimensionss'].setValue(imath.V3f(2))"})["result"] + assert bad["status"] == "failed" and bad["restored"] is True, bad + assert "dimensions" in bad["hint"], bad + typed = ask("execute_code", {"script_handle": handle, "code": "script['box']['dimensions'].setValue('big')"})["result"] + assert "V3fPlug.setValue takes a imath.V3f" in typed["hint"], typed + computed = ask("execute_code", {"script_handle": handle, "code": "script['e'] = Gaffer.Expression()\nscript['e'].setExpression('parent[\"box\"][\"dimensions\"][\"x\"] = 1 / 0', 'python')\nscript['box']['out'].bound('/cube')"})["result"] + assert computed["restored"] is True and "Its python expression" in computed["hint"], computed + read = ask("execute_code", {"script_handle": handle, "read_only": True, "code": "result = {'nodes': [n.getName() for n in script.children(Gaffer.Node)]}"})["result"] + assert read["changed"] is False and "plane" not in read["result"]["nodes"] and "e" not in read["result"]["nodes"], read + evidence["hints"] = {"child": bad["hint"], "type": typed["hint"], "compute": computed["hint"]} + checks.append("code") + + # A change outside the undo step: the rollback cannot give the script back, which is quarantined until the + # agent read it and releases it with that read's revision. + leak = ask("execute_code", {"script_handle": handle, "code": "with Gaffer.UndoScope(script, Gaffer.UndoScope.State.Disabled):\n script['leak'] = Gaffer.Node()\nscript['tracked'] = Gaffer.Node()\nraise ValueError('late')"})["result"] + assert leak["restored"] is False and leak["quarantine"]["cause"].startswith("failed code"), leak + refused = ask("execute_code", {"script_handle": handle, "code": "script['more'] = Gaffer.Node()"}) + assert refused["error"]["code"] == "quarantined", refused + assert ask("release", {"script_handle": handle, "revision": 1})["error"]["code"] == "stale_revision" + revision = ask("context", {"script_handle": handle, "limit": 1})["result"]["revision"] + assert ask("release", {"script_handle": handle, "revision": revision})["result"]["released"] is True + assert ask("execute_code", {"script_handle": handle, "code": "script['more'] = Gaffer.Node()"})["result"]["status"] == "executed" + checks.append("quarantine") + + # The main thread busy: status answers at once, another request is refused within 10 s, saying what runs. + long = [] + runner = threading.Thread(target=lambda: long.append(ask("execute_code", {"script_handle": handle, "read_only": True, "code": "import time\ntime.sleep(14)\nresult = 1"}))) + runner.start() + time.sleep(1.5) + status = ask("status", {})["result"] + assert status["busy"] and status["running"] == "execute_code", status + started = time.monotonic() + busy = ask("context", {"script_handle": handle, "limit": 1}) + waited = time.monotonic() - started + assert busy["ok"] is False and busy["error"]["code"] == "busy" and "execute_code" in busy["error"]["message"], busy + assert 9 < waited < 12, waited + runner.join(30) + assert long and long[0]["result"]["status"] == "executed", long + assert ask("hello", {"script_handle": handle})["ok"] + evidence["busy"] = {"status": status, "seconds": round(waited, 1), "message": busy["error"]["message"]} + checks.append("gate") except Exception as error: - wire.append(type(error).__name__) + import traceback + checks.append("{}: {}".format(type(error).__name__, error)) + traceback.print_exc() finally: GafferUI.EventLoop.executeOnUIThread(loop.stop) + + threading.Thread(target=probe, daemon=True).start() loop.start() -assert wire == ["ok"], wire +GafferOpenCode.shutdown() +print("GAFFER_EVIDENCE " + json.dumps(evidence)) +assert checks == ["wire", "code", "quarantine", "gate"], checks print("GAFFER_WIRE_OK") +print("GAFFER_CODE_OK") +print("GAFFER_GATE_OK") diff --git a/integrations/gaffer/test/real-host.test.ts b/integrations/gaffer/test/real-host.test.ts index 67ea923..4a7a778 100644 --- a/integrations/gaffer/test/real-host.test.ts +++ b/integrations/gaffer/test/real-host.test.ts @@ -6,7 +6,7 @@ import { join } from "node:path" import { it } from "node:test" import { promisify } from "node:util" -it("real Gaffer: hello, bounded graph read, atomic edit, idempotency, stale revision and undo", { +it("real Gaffer: typed apply, offers per script, hello and status off the main thread, execute_code with rollback, hints and quarantine, busy within 10 s", { skip: !process.env.GAFFER_TEST_BIN, }, async () => { const scratch = mkdtempSync(join(tmpdir(), "gaffer-real-")) @@ -36,9 +36,8 @@ it("real Gaffer: hello, bounded graph read, atomic edit, idempotency, stale revi timeout: 90_000, maxBuffer: 4 * 1024 * 1024, }) - assert.match(stdout, /GAFFER_REAL_OK/) - assert.match(stdout, /GAFFER_PANEL_OK/) - assert.match(stdout, /GAFFER_WIRE_OK/) + for (const marker of ["REAL", "PANEL", "OFFERS", "WIRE", "CODE", "GATE"]) + assert.match(stdout, new RegExp(`GAFFER_${marker}_OK`), stdout.slice(-4000)) } finally { rmSync(scratch, { recursive: true, force: true }) } diff --git a/integrations/gaffer/test/registry.test.ts b/integrations/gaffer/test/registry.test.ts index ecfe316..4c1e242 100644 --- a/integrations/gaffer/test/registry.test.ts +++ b/integrations/gaffer/test/registry.test.ts @@ -1,13 +1,20 @@ // An upgrade without restarting OpenCode: the daemon loaded an older OpenCode for Gaffer, whose offer list lives -// on globalThis under the application's name alone, then this version in the same process. This version's list -// never reuses that one, which judges hellos by the older range and calls this version's host incompatible. +// on globalThis under the application's name alone, then this version in the same process. This version's list, +// named after its host contract, never reuses that one, which judges hellos by the older range and calls this +// version's host incompatible. import assert from "node:assert/strict" +import { randomBytes } from "node:crypto" +import { type AddressInfo, createServer } from "node:net" import { after, it } from "node:test" -import { startFakeHost } from "../../../packages/testing/src/index.ts" -import { readPackage } from "../plugin/src/integration.ts" -import { createOfferRegistry, offerRegistryKey, offerRegistryName } from "../plugin/src/kit.ts" +import { + createOfferRegistry, + offerRegistryKey, + offerRegistryName, +} from "../../../packages/integration-kit/src/index.ts" +import { capabilityHash, startFakeHost } from "../../../packages/testing/src/index.ts" +import { createRegistry, readPackage } from "../plugin/src/integration.ts" import { loadPlugin } from "./helpers.ts" const marker = readPackage().opencodeIntegration @@ -19,22 +26,31 @@ after(() => { for (const key of keys) delete store[key] }) -const announce = (fake: Awaited>, offerKey: string) => ({ +const announce = ( + fake: Awaited>, + offerKey: string, + scriptHandle = fake.scriptHandle, +) => ({ ...fake.endpoint, - script_handle: fake.scriptHandle, - label: "Gaffer", + script_handle: scriptHandle, + label: "Gaffer: shot.gfr", protocol: 1, offer_key: offerKey, }) it("a plugin loaded after an older version in the same process judges its own host ok", async () => { - const fake = await startFakeHost({ hostVersion: readPackage().version, appVersion: "1.7.1.0" }) + const fake = await startFakeHost({ hostVersion: readPackage().version, appVersion: "1.6.15.0" }) // What an older version created when it loaded: the kit's shared list under "gaffer", with its own range. - const older = createOfferRegistry({ expected: { ...expected, range: "^0.1.0" } }) + const older = createOfferRegistry({ expected: { ...expected, range: "^0.2.0" } }) store[offerRegistryKey("gaffer")] = { generation: 2, registry: older } const plugin = await loadPlugin({ shared: true }) try { assert.notEqual(plugin.registry, older, "this version keeps its own list") + assert.equal( + (store[offerRegistryKey(offerRegistryName("gaffer", expected))] as { registry?: unknown })?.registry, + plugin.registry, + "named after this version's contract", + ) assert.deepEqual(await plugin.rpc("advertise", announce(fake, "after-upgrade")), { outcome: "accepted" }) assert.deepEqual( plugin.registry.hosts().offers.map((offer) => offer.protocol), @@ -48,16 +64,101 @@ it("a plugin loaded after an older version in the same process judges its own ho } }) +// One Gaffer, one listener, several scripts: hello answers the script it is asked for (as Host.quickHello). +async function gafferWithScripts(scripts: string[]) { + const token = randomBytes(32).toString("hex") + const capabilities = { + methods: ["hello", "context", "apply"], + operations: [], + undo: "transactional", + background: false, + } + const asked: unknown[] = [] + const server = createServer((socket) => { + socket.setEncoding("utf8") + let buffer = "" + socket.on("data", (chunk: string) => { + buffer += chunk + if (!buffer.includes("\n")) return + const request = JSON.parse(buffer.slice(0, buffer.indexOf("\n"))) + const handle = request.params?.script_handle + asked.push(handle) + const reply = (body: unknown) => socket.end(`${JSON.stringify(body)}\n`) + if (request.token !== token) return reply({ id: request.id, ok: false, error: { code: "unauthorized" } }) + if (!scripts.includes(handle)) + return reply({ id: request.id, ok: false, error: { code: "unknown_script", message: "Unknown script_handle" } }) + reply({ + id: request.id, + ok: true, + result: { + protocol: 1, + host_version: readPackage().version, + app_version: "1.6.15.0", + capabilities, + capability_hash: capabilityHash(capabilities as never), + script_handle: handle, + revision: 0, + document: handle, + }, + }) + }) + socket.on("error", () => {}) + }) + await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)) + const endpoint = { host: "127.0.0.1", port: (server.address() as AddressInfo).port, token } + return { endpoint, asked, close: () => new Promise((resolve) => server.close(() => resolve())) } +} + +it("the list asks hello for each announced script: one Gaffer offers each of its scripts", async () => { + delete store[offerRegistryKey(offerRegistryName("gaffer", expected))] + const gaffer = await gafferWithScripts(["gaffer:one", "gaffer:two"]) + const registry = createRegistry(marker) + const offer = (script_handle: string, offer_key: string, label: string) => ({ + ...gaffer.endpoint, + script_handle, + label, + protocol: 1, + offer_key, + }) + try { + assert.deepEqual(await registry.advertise(offer("gaffer:one", "gaffer-1", "Gaffer: one.gfr")), { + outcome: "accepted", + }) + assert.deepEqual(await registry.advertise(offer("gaffer:two", "gaffer-2", "Gaffer: two.gfr")), { + outcome: "accepted", + }) + // A script this Gaffer does not have (closed since): its hello fails, the offer is refused. + assert.deepEqual(await registry.advertise(offer("gaffer:closed", "gaffer-3", "Gaffer: gone.gfr")), { + outcome: "refused", + }) + assert.deepEqual(gaffer.asked, ["gaffer:one", "gaffer:two", "gaffer:closed"]) + assert.deepEqual( + registry + .hosts() + .offers.map((item) => [item.label, item.protocol]) + .sort(), + [ + ["Gaffer: one.gfr", "ok"], + ["Gaffer: two.gfr", "ok"], + ], + ) + } finally { + registry.retire() + delete store[offerRegistryKey(offerRegistryName("gaffer", expected))] + await gaffer.close() + } +}) + it("a tool's error carries the host's own message and data", async () => { const fake = await startFakeHost({ hostVersion: readPackage().version, - appVersion: "1.7.1.0", + appVersion: "1.6.15.0", methods: { context: () => { throw { - code: "invalid_args", - message: "SceneView renderer is not registered: Arnold", - data: { reason: "unsupported_renderer", registered_renderers: ["OpenGL", "Cycles"] }, + code: "stale_revision", + message: "The Gaffer script changed after the intent was prepared: /Sphere", + data: { reason: "revision_conflict", expected_revision: 3, current_revision: 5 }, } }, }, @@ -67,9 +168,12 @@ it("a tool's error carries the host's own message and data", async () => { await plugin.rpc("advertise", announce(fake, "errors")) const offerID = plugin.registry.hosts().offers[0]?.offerID as string assert.deepEqual(await plugin.rpc("attachHost", { offerID, sessionID: "ses_test" }), { outcome: "attached" }) - await assert.rejects(plugin.tool("gaffer_context", { script_handle: fake.scriptHandle }), (error: Error) => { - assert.match(error.message, /\(invalid_args\): SceneView renderer is not registered: Arnold/) - assert.match(error.message, /"reason":"unsupported_renderer","registered_renderers":\["OpenGL","Cycles"\]/) + await assert.rejects(plugin.tool("gaffer_context"), (error: Error) => { + assert.match( + error.message, + /^The script changed since your last read: call gaffer_context and use its revision\. The Gaffer script changed after the intent was prepared: \/Sphere /, + ) + assert.match(error.message, /"reason":"revision_conflict","expected_revision":3,"current_revision":5/) return true }) } finally { diff --git a/integrations/gaffer/test/saved-verify.test.ts b/integrations/gaffer/test/saved-verify.test.ts new file mode 100644 index 0000000..2999a52 --- /dev/null +++ b/integrations/gaffer/test/saved-verify.test.ts @@ -0,0 +1,124 @@ +// gaffer_verify saved follows its background Gaffer outside Gaffer's main thread: a result, a crash, or +// timeout_s, after which it says where the background Gaffer was. + +import assert from "node:assert/strict" +import { existsSync, mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs" +import { tmpdir } from "node:os" +import { join } from "node:path" +import { it } from "node:test" + +import { isSavedStart, type SavedStart, waitSaved } from "../plugin/src/saved-verify.ts" + +const step = (what: Record) => `OPENCODE_STEP ${JSON.stringify(what)}` + +function started(log: string): SavedStart { + const work = mkdtempSync(join(tmpdir(), "gaffer-saved-")) + mkdirSync(work, { recursive: true }) + writeFileSync(join(work, "verify.log"), log) + return { + pid: 4242, + work, + log: join(work, "verify.log"), + result: join(work, "result.json"), + file: "D:/shots/shot.gfr", + saved_at: "2026-11-16 07:25:51", + bytes: 48213, + timeout_s: 5, + } +} + +// A clock that advances with each wait: no real waiting. +function clock() { + let t = 1_000_000 + return { + now: () => t, + sleep: async (ms: number) => { + t += ms + }, + } +} + +it("a verify that runs out of time is stopped and says what it was doing", async () => { + const start = started( + [ + step({ step: "load", file: "D:/shots/shot.gfr", at_s: 0 }), + step({ step: "loaded", seconds: 0.4, at_s: 0.4 }), + ...Array.from({ length: 11 }, (_, i) => + step({ step: "criterion", index: i + 1, of: 16, label: `check ${i + 1}`, frame: 48, at_s: 0.5 + i * 0.1 }), + ), + ].join("\n"), + ) + const stopped: number[] = [] + try { + assert.ok(isSavedStart(start)) + await assert.rejects( + waitSaved(start, { ...clock(), alive: () => true, stop: (pid) => stopped.push(pid) }), + (error: Error) => { + assert.match(error.message, /gaffer_verify saved stopped the background Gaffer after 5 s \(timeout_s\)/) + assert.match(error.message, /it was measuring criterion 11\/16 "check 11" at frame 48/) + assert.match(error.message, /The file loaded in 0\.4 s; 10 criteria were measured before\./) + assert.match(error.message, /give more time with timeout_s \(up to 3600\)/) + return true + }, + ) + assert.deepEqual(stopped, [4242]) + assert.equal(existsSync(start.work), false) + } finally { + rmSync(start.work, { recursive: true, force: true }) + } +}) + +it("a verify that finishes gives its result and what it read", async () => { + const start = started(`${step({ step: "load", at_s: 0 })}\n${step({ step: "done", at_s: 1 })}`) + writeFileSync(start.result, JSON.stringify({ passed: 16, failed: 0, load_seconds: 0.4, load_warnings: [] })) + try { + const result = (await waitSaved(start, { ...clock(), alive: () => true })) as { + passed: number + read: Record + } + assert.equal(result.passed, 16) + assert.equal(result.read.file, "D:/shots/shot.gfr") + assert.equal(result.read.load_seconds, 0.4) + assert.match(String(result.read.by), /background Gaffer on the file as written on disk, not the open script/) + assert.equal("load_warnings" in result.read, false) + assert.equal(existsSync(start.work), false) + } finally { + rmSync(start.work, { recursive: true, force: true }) + } +}) + +it("a verify whose background Gaffer crashed says so, with where it was", async () => { + const start = started( + [ + step({ step: "loaded", seconds: 1, at_s: 1 }), + step({ step: "render", at_s: 2 }), + "Fatal error: Segmentation fault", + "CURRENT THREAD 1", + "+0x1 [IECoreGL::Renderer::render] libIECoreGL.so", + ].join("\n"), + ) + try { + await assert.rejects(waitSaved(start, { ...clock(), alive: () => false }), (error: Error) => { + assert.match( + error.message, + /crashed \(Segmentation fault in IECoreGL::Renderer::render\) while rendering the check image; the open Gaffer is not affected/, + ) + return true + }) + } finally { + rmSync(start.work, { recursive: true, force: true }) + } +}) + +it("a verify that reports an error says the file it read", async () => { + const start = started(step({ step: "loaded", seconds: 1, at_s: 1 })) + writeFileSync(start.result, JSON.stringify({ error: "KeyError: 'Sphere'" })) + try { + await assert.rejects( + waitSaved(start, { ...clock(), alive: () => true }), + /verify in the background Gaffer reading D:\/shots\/shot\.gfr failed:\nKeyError: 'Sphere'/, + ) + } finally { + rmSync(start.work, { recursive: true, force: true }) + } +}) diff --git a/integrations/gaffer/test/tasks.test.ts b/integrations/gaffer/test/tasks.test.ts new file mode 100644 index 0000000..2897ee8 --- /dev/null +++ b/integrations/gaffer/test/tasks.test.ts @@ -0,0 +1,121 @@ +// Render tasks: a background Gaffer that crashed is said as a crash, with its first frames, and with the render +// program's risk note when it had one; a job that says it finished without its files is a failure. + +import assert from "node:assert/strict" +import { spawn } from "node:child_process" +import { mkdtempSync, rmSync, writeFileSync } from "node:fs" +import { tmpdir } from "node:os" +import { join } from "node:path" +import { it } from "node:test" + +import { isAlive } from "../plugin/src/kit.ts" +import { addTask, cancelTask, crashOf, isRenderStart, taskStatus } from "../plugin/src/tasks.ts" + +// The head of a background Gaffer render log whose renderer crashed. +const CRASHED = [ + 'OPENCODE_RENDER {"node": "/Render", "type": "GafferCycles::CyclesRender", "frames": [1, 1], "count": 1}', + "Fatal error: Segmentation fault", + "CURRENT THREAD 6560", + "+0x7ff8 [ccl::BVH2::build] libcycles.so", + "+0x7ff9 [ccl::Device::build_bvh] libcycles.so", + "+0x7ffa [IECoreCycles::Renderer::render] libIECoreCycles.so", + "+0x7ffb [ccl::BVH2::build] libcycles.so", +].join("\n") + +function started(dir: string, log: string, extra: Record = {}) { + const file = join(dir, "render.log") + writeFileSync(file, log) + return addTask({ + // This test process, alive: a crash is said even while the crashed Gaffer is still being torn down. + pid: process.pid, + log: file, + kind: "render", + targets: [join(dir, "beauty.0001.exr")], + node: "/Render", + start: 1, + ...extra, + }) +} + +it("a crashed render Gaffer is said as a crash with its first frames and the render's risk note", () => { + assert.equal( + crashOf(CRASHED), + "Segmentation fault in ccl::BVH2::build < ccl::Device::build_bvh < IECoreCycles::Renderer::render", + ) + assert.equal(crashOf("Caught signal 11\n"), "signal 11") + assert.equal(crashOf("OPENCODE_PROGRESS {}\nfine"), undefined) + const dir = mkdtempSync(join(tmpdir(), "gaffer-task-")) + try { + const task = started(dir, CRASHED, { facts: { crash_risk: "This scene has 3 million instances" } }) + const status = taskStatus(task) as { state: string; error?: string; kind: string; frames?: number[] } + assert.equal(status.kind, "render") + assert.equal(status.state, "failed") + assert.deepEqual(status.frames, [1, 1]) + assert.match( + String(status.error), + /^the background Gaffer crashed \(Segmentation fault in ccl::BVH2::build < .*\); the open Gaffer is not affected\. This scene has 3 million instances\. See /, + ) + } finally { + rmSync(dir, { recursive: true, force: true }) + } +}) + +it("a render is done only when it says so and every expected file is there, not empty", () => { + const dir = mkdtempSync(join(tmpdir(), "gaffer-task-")) + try { + const rendering = started(dir, 'OPENCODE_PROGRESS {"done": 0, "total": 1}\n') + assert.deepEqual( + [taskStatus(rendering).state, taskStatus(rendering).progress], + ["rendering", { done: 0, total: 1 }], + ) + const claimed = started(dir, 'OPENCODE_DONE {"output": "beauty"}\n') + const failed = taskStatus(claimed) as { state: string; error?: string } + assert.equal(failed.state, "failed") + assert.match(String(failed.error), /1 of 1 expected file\(s\) are missing or empty: beauty\.0001\.exr/) + writeFileSync(join(dir, "beauty.0001.exr"), "exr") + const done = taskStatus(claimed) as { state: string; result?: unknown } + assert.equal(done.state, "done") + assert.deepEqual(done.result, { output: "beauty" }) + // Only render tasks: a PDG-like start is not one. + assert.equal(isRenderStart({ pid: 1, log: "x", kind: "pdg", targets: [] }), false) + assert.equal(isRenderStart({ pid: 1, log: "x", kind: "render", targets: [] }), true) + } finally { + rmSync(dir, { recursive: true, force: true }) + } +}) + +// gaffer.cmd runs gaffer.exe under cmd.exe: cancelling the task (the launcher's pid) stops the process it started too. +it("a cancelled render stops the launcher and the process it started", { + skip: process.platform !== "win32", +}, async () => { + const dir = mkdtempSync(join(tmpdir(), "gaffer-cancel-")) + try { + const log = join(dir, "render.log") + writeFileSync(log, 'OPENCODE_RENDER {"node": "/ImageWriter"}\n') + const launcher = spawn("cmd.exe", ["/d", "/c", "ping -n 60 127.0.0.1 > nul"], { windowsHide: true }) + await new Promise((done) => setTimeout(done, 1_000)) + const { execFileSync } = await import("node:child_process") + const children = execFileSync( + "powershell.exe", + ["-NoProfile", "-Command", `(Get-CimInstance Win32_Process -Filter "ParentProcessId=${launcher.pid}").ProcessId`], + { encoding: "utf8", windowsHide: true }, + ) + .split(/\s+/) + .filter(Boolean) + .map(Number) + assert.ok(children.length > 0, "the launcher started its process") + const task = addTask({ + pid: launcher.pid as number, + log, + kind: "render", + targets: [join(dir, "x.exr")], + node: "/ImageWriter", + }) + assert.equal((cancelTask(task) as { state: string }).state, "cancelled") + await new Promise((done) => setTimeout(done, 500)) + assert.equal(isAlive(launcher.pid as number), false) + for (const child of children) assert.equal(isAlive(child), false, `process ${child} still runs`) + } finally { + rmSync(dir, { recursive: true, force: true }) + } +})