fix

apps/api/internal/processing/openai.go

@@ -7,6 +7,7 @@ import (
 	"errors"
 	"fmt"
 	"io"
+	"log"
 	"math"
 	"math/rand"
 	"net"
@@ -39,6 +40,16 @@ type OpenAIClient struct {
 
 const maxOpenAIRetries = 8
 
+// openAIHTTPTimeout is the full-request limit for OpenAI-compatible chat/embeddings.
+// Reasoning models (e.g. code-fast) often need 60–180s; 240s reduces false timeouts.
+// Timeouts are not retried (see classifyOpenAITransportErr) so a hung upstream fails
+// the item in ~4m instead of MaxRetries×240s of silent PROCESSING.
+const openAIHTTPTimeout = 240 * time.Second
+
+// openAIWaitLogInterval emits progress while HTTPClient.Do blocks (cancel is only
+// checked between job items, so mid-call silence looks like a stuck job).
+const openAIWaitLogInterval = 30 * time.Second
+
 func NewOpenAIClient(apiKey, baseURL, model string, rpm, maxRetries int) *OpenAIClient {
 	if baseURL == "" {
 		baseURL = "https://api.openai.com/v1"
@@ -57,14 +68,12 @@ func NewOpenAIClient(apiKey, baseURL, model string, rpm, maxRetries int) *OpenAI
 	}
 	// Dial-time SSRF. Loopback when base URL is local; RFC1918 only in non-prod.
 	policy := openAIDialPolicy(baseURL)
-	// Header/body timeout: code-fast often exceeds 180s; 240s reduces false timeouts.
-	// Tradeoff: slower fail on hung upstream. Prefer synthesize fallback over infinite wait
-	// (RunSteps always synthesizes on timeout/error/empty — do not raise unboundedly).
+	// Prefer synthesize fallback on timeout/error/empty (RunSteps) over raising this unboundedly.
 	return &OpenAIClient{
 		APIKey:      apiKey,
 		BaseURL:     strings.TrimRight(baseURL, "/"),
 		Model:       model,
-		HTTPClient:  security.SafeHTTPClientPolicy(240*time.Second, policy),
+		HTTPClient:  security.SafeHTTPClientPolicy(openAIHTTPTimeout, policy),
 		MinInterval: interval,
 		MaxRetries:  maxRetries,
 		ModeLabel:   AIProviderInternal,
@@ -141,6 +150,79 @@ func IsMockOrLoopbackBaseURL(baseURL string) bool {
 	return strings.Contains(lower, "mock-llm")
 }
 
+// startWaitLogger logs request start and periodic progress while Do blocks.
+// Call the returned stop with the Do error (or nil) when the request finishes.
+func (c *OpenAIClient) startWaitLogger(op string, maxTokens int) func(error) {
+	if c == nil {
+		return func(error) {}
+	}
+	start := time.Now()
+	log.Printf("openai: request start op=%s model=%s base=%s max_tokens=%d timeout=%s",
+		op, c.Model, c.BaseURL, maxTokens, openAIHTTPTimeout)
+	done := make(chan struct{})
+	var once sync.Once
+	go func() {
+		t := time.NewTicker(openAIWaitLogInterval)
+		defer t.Stop()
+		for {
+			select {
+			case <-done:
+				return
+			case <-t.C:
+				log.Printf("openai: still waiting op=%s model=%s base=%s elapsed=%s timeout=%s",
+					op, c.Model, c.BaseURL, time.Since(start).Round(time.Second), openAIHTTPTimeout)
+			}
+		}
+	}()
+	return func(err error) {
+		once.Do(func() {
+			close(done)
+			elapsed := time.Since(start).Round(time.Millisecond)
+			if err != nil {
+				log.Printf("openai: request done op=%s model=%s elapsed=%s err=%s",
+					op, c.Model, elapsed, TruncateError(err))
+				return
+			}
+			log.Printf("openai: request done op=%s model=%s elapsed=%s ok=1", op, c.Model, elapsed)
+		})
+	}
+}
+
+// isOpenAITimeoutErr reports HTTP client / context deadline timeouts.
+// These must not be retried: each attempt costs openAIHTTPTimeout and leaves the job looking stuck.
+func isOpenAITimeoutErr(err error) bool {
+	if err == nil {
+		return false
+	}
+	if errors.Is(err, context.DeadlineExceeded) {
+		return true
+	}
+	var ne net.Error
+	if errors.As(err, &ne) && ne.Timeout() {
+		return true
+	}
+	s := err.Error()
+	return strings.Contains(s, "Client.Timeout exceeded") ||
+		strings.Contains(s, "context deadline exceeded")
+}
+
+// classifyOpenAITransportErr maps Do/Read errors to (wrappedErr, retryable).
+// Timeouts and cancel are fatal for this attempt chain; other network errors may retry.
+func classifyOpenAITransportErr(model string, err error) (error, bool) {
+	if err == nil {
+		return nil, false
+	}
+	if errors.Is(err, context.Canceled) {
+		return err, false
+	}
+	if isOpenAITimeoutErr(err) {
+		return fmt.Errorf(
+			"openai timed out waiting for model %q after %s (not retrying; use a faster model or fix upstream latency): %w",
+			model, openAIHTTPTimeout, err), false
+	}
+	return err, true
+}
+
 type chatRequest struct {
 	Model       string        `json:"model"`
 	Messages    []chatMessage `json:"messages"`
@@ -218,9 +300,18 @@ func (c *OpenAIClient) CompleteWithOptions(ctx context.Context, system, user str
 			return comp, nil
 		}
 		lastErr = err
-		if errors.Is(err, errEmptyModelResponse) && maxTok > 0 && maxTok < maxTokensReasoningBudget {
-			maxTok = maxTokensReasoningBudget
-			retryable = true
+		if errors.Is(err, errEmptyModelResponse) {
+			if maxTok <= 0 || maxTok < maxTokensReasoningBudget {
+				if maxTok < maxTokensReasoningBudget {
+					maxTok = maxTokensReasoningBudget
+				}
+				retryable = true
+			} else {
+				// Already at reasoning budget — another 240s call will not help.
+				return Completion{}, fmt.Errorf(
+					"openai empty response at max_tokens=%d for model %q (try a faster non-reasoning model): %w",
+					maxTok, c.Model, err)
+			}
 		}
 		if !retryable {
 			return Completion{}, err
@@ -325,14 +416,18 @@ func (c *OpenAIClient) doEmbed(ctx context.Context, texts []string) ([][]float32
 	req.Header.Set("Content-Type", "application/json")
 	req.Header.Set("Authorization", "Bearer "+c.APIKey)
 
+	stopWait := c.startWaitLogger("embeddings", 0)
 	res, err := c.HTTPClient.Do(req)
+	stopWait(err)
 	if err != nil {
-		return nil, true, err
+		wrapped, retryable := classifyOpenAITransportErr(c.Model, err)
+		return nil, retryable, wrapped
 	}
 	defer res.Body.Close()
 	raw, err := io.ReadAll(io.LimitReader(res.Body, 4<<20))
 	if err != nil {
-		return nil, true, err
+		wrapped, retryable := classifyOpenAITransportErr(c.Model, err)
+		return nil, retryable, wrapped
 	}
 	var parsed embeddingResponse
 	if err := json.Unmarshal(raw, &parsed); err != nil {
@@ -391,14 +486,18 @@ func (c *OpenAIClient) doComplete(ctx context.Context, system, user string, temp
 	req.Header.Set("Content-Type", "application/json")
 	req.Header.Set("Authorization", "Bearer "+c.APIKey)
 
+	stopWait := c.startWaitLogger("chat/completions", maxTokens)
 	res, err := c.HTTPClient.Do(req)
+	stopWait(err)
 	if err != nil {
-		return Completion{}, true, err
+		wrapped, retryable := classifyOpenAITransportErr(c.Model, err)
+		return Completion{}, retryable, wrapped
 	}
 	defer res.Body.Close()
 	raw, err := io.ReadAll(io.LimitReader(res.Body, 1<<20))
 	if err != nil {
-		return Completion{}, true, err
+		wrapped, retryable := classifyOpenAITransportErr(c.Model, err)
+		return Completion{}, retryable, wrapped
 	}
 	var parsed chatResponse
 	if err := json.Unmarshal(raw, &parsed); err != nil {
@@ -426,7 +525,9 @@ func (c *OpenAIClient) doComplete(ctx context.Context, system, user string, temp
 	}
 	if text == "" {
 		// Reasoning models often return empty content when max_tokens cuts mid-thought.
-		retryable := strings.EqualFold(finishReason, "length")
+		// Only retry when CompleteWithOptions can still raise max_tokens.
+		canBump := maxTokens <= 0 || maxTokens < maxTokensReasoningBudget
+		retryable := strings.EqualFold(finishReason, "length") && canBump
 		return Completion{}, retryable, errEmptyModelResponse
 	}
 	return Completion{