From 8d3d1474f8d9d869241336adbb074455e8489e3b Mon Sep 17 00:00:00 2001 From: Jan Svabenik Date: Thu, 21 May 2026 16:27:09 +0200 Subject: [PATCH] feat: AI-Automapping fuer Semantic Slots + manueller Override - ManifestField bekommt semantic_slot/mapping_source/mapping_confidence (Migration 009, Domain-Model, SQLite-Store). - Neuer AISlotMapper ruft den aktiven LLM-Provider mit cleverem Prompt: klassifiziert jedes Textfeld in einen festen Slot-Katalog (hero/intro/ about/cta/service_items[N]/team_items[N]/testimonial_items[N]) anhand Section, Block-ID, displayOrder und Sample-Wert. - Onboarding ruft den Mapper nach Discovery auf und persistiert die Slots als source=ai. Fehler werden gelogged, Onboarding bleibt erfolgreich. - MapTemplateFieldsToSemanticSlots respektiert persistierte Slots; die Rule-basierte Logik laeuft nur fuer Felder ohne Slot weiter. - Template-Detail-UI hat neue Dropdown-Spalte "Semantic Slot"; manueller Override setzt source=manual. Custom-Werte (ausserhalb der Liste) bleiben sichtbar und werden nicht versehentlich ueberschrieben. Co-Authored-By: Claude Sonnet 4.6 --- internal/app/app.go | 3 +- internal/domain/models.go | 37 +- internal/httpserver/handlers/ui.go | 97 ++++- internal/mapping/ai_slot_mapper.go | 362 ++++++++++++++++++ internal/mapping/semantic_slots.go | 26 +- internal/onboarding/service.go | 64 ++++ .../009_add_semantic_slot_to_fields.sql | 8 + internal/store/sqlite/store.go | 18 +- web/templates/template_detail.gohtml | 12 + 9 files changed, 586 insertions(+), 41 deletions(-) create mode 100644 internal/mapping/ai_slot_mapper.go create mode 100644 internal/store/sqlite/migrations/009_add_semantic_slot_to_fields.sql diff --git a/internal/app/app.go b/internal/app/app.go index 0d4e88e..73a0399 100644 --- a/internal/app/app.go +++ b/internal/app/app.go @@ -71,11 +71,12 @@ func New(cfg config.Config) (*App, error) { qc := qcclient.New(cfg.QCBaseURL, cfg.QCToken, 15*time.Second, logger) templateSvc := templatesvc.New(qc, templateStore, manifestStore) - onboardSvc := onboarding.New(qc, templateStore, manifestStore) draftSvc := draftsvc.New(draftStore, templateStore, manifestStore) mappingSvc := mapping.New() buildSvc := buildsvc.New(qc, templateStore, manifestStore, buildStore, mappingSvc, time.Duration(cfg.PollTimeoutSeconds)*time.Second) providerRuntime := llmruntime.NewFactory(45 * time.Second) + onboardSvc := onboarding.New(qc, templateStore, manifestStore). + WithAISlotMapper(mapping.NewAISlotMapper(settingsStore, providerRuntime)) suggestionGenerator := mapping.NewCompositeSuggestionGenerator( mapping.NewGooglePlacesSuggestionGenerator(settingsStore), mapping.NewCompositeSuggestionGenerator( diff --git a/internal/domain/models.go b/internal/domain/models.go index ed0ebe4..e96196f 100644 --- a/internal/domain/models.go +++ b/internal/domain/models.go @@ -36,22 +36,31 @@ type TemplateManifest struct { } type TemplateField struct { - ID string `json:"id"` - TemplateID int64 `json:"templateId"` - ManifestID string `json:"manifestId"` - Section string `json:"section"` - WebsiteSection string `json:"websiteSection"` - KeyName string `json:"keyName"` - Path string `json:"path"` - FieldKind string `json:"fieldKind"` - SampleValue string `json:"sampleValue"` - IsEnabled bool `json:"isEnabled"` - IsRequiredByUs bool `json:"isRequiredByUs"` - DisplayLabel string `json:"displayLabel"` - DisplayOrder int `json:"displayOrder"` - Notes string `json:"notes"` + ID string `json:"id"` + TemplateID int64 `json:"templateId"` + ManifestID string `json:"manifestId"` + Section string `json:"section"` + WebsiteSection string `json:"websiteSection"` + KeyName string `json:"keyName"` + Path string `json:"path"` + FieldKind string `json:"fieldKind"` + SampleValue string `json:"sampleValue"` + IsEnabled bool `json:"isEnabled"` + IsRequiredByUs bool `json:"isRequiredByUs"` + DisplayLabel string `json:"displayLabel"` + DisplayOrder int `json:"displayOrder"` + Notes string `json:"notes"` + SemanticSlot string `json:"semanticSlot,omitempty"` + MappingSource string `json:"mappingSource,omitempty"` + MappingConfidence float64 `json:"mappingConfidence,omitempty"` } +const ( + MappingSourceRule = "rule" + MappingSourceAI = "ai" + MappingSourceManual = "manual" +) + type SiteBuild struct { ID string `json:"id"` TemplateID int64 `json:"templateId"` diff --git a/internal/httpserver/handlers/ui.go b/internal/httpserver/handlers/ui.go index 7e5fd6c..7efdab6 100644 --- a/internal/httpserver/handlers/ui.go +++ b/internal/httpserver/handlers/ui.go @@ -86,15 +86,19 @@ type templatesPageData struct { } type templateFieldView struct { - Path string - FieldKind string - IsEnabled bool - IsRequiredByUs bool - DisplayLabel string - DisplayOrder int - WebsiteSection string - Notes string - SampleValue string + Path string + FieldKind string + IsEnabled bool + IsRequiredByUs bool + DisplayLabel string + DisplayOrder int + WebsiteSection string + Notes string + SampleValue string + SemanticSlot string + SemanticSlotIsKnown bool + MappingSource string + MappingConfidence float64 } type websiteSectionOptionView struct { @@ -102,11 +106,17 @@ type websiteSectionOptionView struct { Label string } +type semanticSlotOptionView struct { + Value string + Label string +} + type templateDetailPageData struct { pageData Detail *templatesvc.TemplateDetail Fields []templateFieldView WebsiteSectionOptions []websiteSectionOptionView + SemanticSlotOptions []semanticSlotOptionView } type buildFieldView struct { @@ -359,18 +369,28 @@ func (u *UI) TemplateDetail(w http.ResponseWriter, r *http.Request) { http.Error(w, err.Error(), http.StatusNotFound) return } + slotOpts := semanticSlotOptions() + knownSlots := make(map[string]bool, len(slotOpts)) + for _, opt := range slotOpts { + knownSlots[opt.Value] = true + } fields := make([]templateFieldView, 0, len(detail.Fields)) for _, f := range detail.Fields { + slot := strings.TrimSpace(f.SemanticSlot) fields = append(fields, templateFieldView{ - Path: f.Path, - FieldKind: f.FieldKind, - IsEnabled: f.IsEnabled, - IsRequiredByUs: f.IsRequiredByUs, - DisplayLabel: f.DisplayLabel, - DisplayOrder: f.DisplayOrder, - WebsiteSection: domain.NormalizeWebsiteSection(f.WebsiteSection), - Notes: f.Notes, - SampleValue: f.SampleValue, + Path: f.Path, + FieldKind: f.FieldKind, + IsEnabled: f.IsEnabled, + IsRequiredByUs: f.IsRequiredByUs, + DisplayLabel: f.DisplayLabel, + DisplayOrder: f.DisplayOrder, + WebsiteSection: domain.NormalizeWebsiteSection(f.WebsiteSection), + Notes: f.Notes, + SampleValue: f.SampleValue, + SemanticSlot: slot, + SemanticSlotIsKnown: slot == "" || knownSlots[slot], + MappingSource: f.MappingSource, + MappingConfidence: f.MappingConfidence, }) } u.render.Render(w, "template_detail", templateDetailPageData{ @@ -378,6 +398,7 @@ func (u *UI) TemplateDetail(w http.ResponseWriter, r *http.Request) { Detail: detail, Fields: fields, WebsiteSectionOptions: websiteSectionOptions(), + SemanticSlotOptions: slotOpts, }) } @@ -420,6 +441,7 @@ func (u *UI) UpdateTemplateFields(w http.ResponseWriter, r *http.Request) { http.Redirect(w, r, fmt.Sprintf("/templates/%d?err=%s", templateID, urlQuery("invalid display order")), http.StatusSeeOther) return } + semanticSlot := strings.TrimSpace(r.FormValue(fmt.Sprintf("field_semantic_slot_%d", i))) patches = append(patches, onboarding.FieldPatch{ Path: path, IsEnabled: boolPtr(enabled), @@ -428,6 +450,7 @@ func (u *UI) UpdateTemplateFields(w http.ResponseWriter, r *http.Request) { DisplayOrder: intPtr(order), WebsiteSection: strPtr(websiteSection), Notes: strPtr(notes), + SemanticSlot: strPtr(semanticSlot), }) } @@ -2152,3 +2175,41 @@ func websiteSectionOptions() []websiteSectionOptionView { } return out } + +func semanticSlotOptions() []semanticSlotOptionView { + // "" = automatisch (rule-basiertes Fallback). Reihenfolge ist UI-relevant. + slots := []semanticSlotOptionView{ + {Value: "", Label: "(automatisch)"}, + {Value: "hero.title", Label: "Hero - Title"}, + {Value: "intro.title", Label: "Intro - Title"}, + {Value: "intro.description", Label: "Intro - Description"}, + {Value: "about.description", Label: "About - Description"}, + {Value: "cta.text", Label: "CTA - Text"}, + {Value: "service_items[0].title", Label: "Service Item #1 - Title"}, + {Value: "service_items[0].description", Label: "Service Item #1 - Description"}, + {Value: "service_items[1].title", Label: "Service Item #2 - Title"}, + {Value: "service_items[1].description", Label: "Service Item #2 - Description"}, + {Value: "service_items[2].title", Label: "Service Item #3 - Title"}, + {Value: "service_items[2].description", Label: "Service Item #3 - Description"}, + {Value: "service_items[3].title", Label: "Service Item #4 - Title"}, + {Value: "service_items[3].description", Label: "Service Item #4 - Description"}, + {Value: "service_items[4].title", Label: "Service Item #5 - Title"}, + {Value: "service_items[4].description", Label: "Service Item #5 - Description"}, + {Value: "team_items[0].name", Label: "Team #1 - Name"}, + {Value: "team_items[0].description", Label: "Team #1 - Description"}, + {Value: "team_items[1].name", Label: "Team #2 - Name"}, + {Value: "team_items[1].description", Label: "Team #2 - Description"}, + {Value: "team_items[2].name", Label: "Team #3 - Name"}, + {Value: "team_items[2].description", Label: "Team #3 - Description"}, + {Value: "testimonial_items[0].title", Label: "Testimonial #1 - Title"}, + {Value: "testimonial_items[0].name", Label: "Testimonial #1 - Name"}, + {Value: "testimonial_items[0].description", Label: "Testimonial #1 - Description"}, + {Value: "testimonial_items[1].title", Label: "Testimonial #2 - Title"}, + {Value: "testimonial_items[1].name", Label: "Testimonial #2 - Name"}, + {Value: "testimonial_items[1].description", Label: "Testimonial #2 - Description"}, + {Value: "testimonial_items[2].title", Label: "Testimonial #3 - Title"}, + {Value: "testimonial_items[2].name", Label: "Testimonial #3 - Name"}, + {Value: "testimonial_items[2].description", Label: "Testimonial #3 - Description"}, + } + return slots +} diff --git a/internal/mapping/ai_slot_mapper.go b/internal/mapping/ai_slot_mapper.go new file mode 100644 index 0000000..67d0451 --- /dev/null +++ b/internal/mapping/ai_slot_mapper.go @@ -0,0 +1,362 @@ +package mapping + +import ( + "context" + "encoding/json" + "fmt" + "regexp" + "sort" + "strings" + "time" + + "qctextbuilder/internal/domain" + "qctextbuilder/internal/llmruntime" +) + +// AISlotMapper classifies template fields into semantic slots via the active +// LLM provider. It is meant to run once per template (during onboarding) and +// its results are persisted on the manifest fields. +type AISlotMapper struct { + settings SettingsReader + runtimeFactory *llmruntime.Factory +} + +func NewAISlotMapper(settings SettingsReader, runtimeFactory *llmruntime.Factory) *AISlotMapper { + return &AISlotMapper{settings: settings, runtimeFactory: runtimeFactory} +} + +type AISlotAssignment struct { + FieldPath string `json:"fieldPath"` + Slot string `json:"slot"` + Confidence float64 `json:"confidence"` + Reason string `json:"reason"` +} + +var aiSlotRepeatedPattern = regexp.MustCompile(`^(service_items|team_items|testimonial_items)\[\d+\]\.(title|description|name)$`) + +var aiSlotSingletons = map[string]struct{}{ + "hero.title": {}, + "intro.title": {}, + "intro.description": {}, + "about.description": {}, + "cta.text": {}, +} + +func isValidSemanticSlot(slot string) bool { + if slot == "" { + return false + } + if _, ok := aiSlotSingletons[slot]; ok { + return true + } + if !aiSlotRepeatedPattern.MatchString(slot) { + return false + } + // team_items[N].title is not a valid combination — name is the only label role. + switch { + case strings.HasPrefix(slot, "team_items[") && strings.HasSuffix(slot, ".title"): + return false + case strings.HasPrefix(slot, "service_items[") && strings.HasSuffix(slot, ".name"): + return false + } + return true +} + +// MapFields returns one assignment per input field whose Slot is a valid +// semantic slot. Fields the model decided to skip are omitted. +func (m *AISlotMapper) MapFields(ctx context.Context, templateID int64, fields []domain.TemplateField) ([]AISlotAssignment, error) { + started := time.Now() + if m == nil || m.settings == nil || m.runtimeFactory == nil { + return nil, fmt.Errorf("ai slot mapper is not configured") + } + + textFields := make([]domain.TemplateField, 0, len(fields)) + for _, f := range fields { + if !f.IsEnabled { + continue + } + if !strings.EqualFold(strings.TrimSpace(f.FieldKind), "text") { + continue + } + textFields = append(textFields, f) + } + if len(textFields) == 0 { + return []AISlotAssignment{}, nil + } + + settings, err := m.settings.GetSettings(ctx) + if err != nil || settings == nil { + return nil, fmt.Errorf("llm settings are not available") + } + provider := domain.NormalizeLLMProvider(settings.LLMActiveProvider) + model := domain.NormalizeLLMModel(provider, settings.LLMActiveModel) + if strings.TrimSpace(model) == "" { + return nil, fmt.Errorf("no active llm model configured") + } + apiKey := domain.LLMAPIKeyForProvider(provider, *settings) + if provider != domain.LLMProviderOllama && strings.TrimSpace(apiKey) == "" { + return nil, fmt.Errorf("api key for provider %s is not configured", provider) + } + baseURL := strings.TrimSpace(settings.LLMBaseURL) + + client, err := m.runtimeFactory.ClientFor(provider) + if err != nil { + return nil, err + } + + systemPrompt, userPrompt := buildAISlotPrompts(textFields) + mappingLogger().InfoContext(ctx, "ai slot mapping", + "component", "automapper", + "step", "request", + "status", "start", + "provider", provider, + "model", model, + "template_id", templateID, + "field_count", len(textFields), + ) + temperature := 0.0 + maxTokens := domain.NormalizeLLMMaxTokens(settings.LLMMaxTokens) + if maxTokens < 2048 { + maxTokens = 2048 + } + raw, err := client.Generate(ctx, llmruntime.Request{ + Provider: provider, + Model: model, + BaseURL: baseURL, + APIKey: strings.TrimSpace(apiKey), + Temperature: &temperature, + MaxTokens: &maxTokens, + SystemPrompt: systemPrompt, + UserPrompt: userPrompt, + }) + if err != nil { + mappingLogger().WarnContext(ctx, "ai slot mapping", + "component", "automapper", + "step", "request", + "status", "failed", + "provider", provider, + "model", model, + "template_id", templateID, + "error", shortErr(err), + "duration_ms", time.Since(started).Milliseconds(), + ) + return nil, fmt.Errorf("ai slot mapper request failed (provider=%s model=%s): %w", provider, model, err) + } + + parsed, err := parseAISlotResponse(raw) + if err != nil { + mappingLogger().WarnContext(ctx, "ai slot mapping", + "component", "automapper", + "step", "parse", + "status", "failed", + "provider", provider, + "model", model, + "template_id", templateID, + "error", shortErr(err), + "response_snippet", providerLogSnippet(raw, 1500), + "duration_ms", time.Since(started).Milliseconds(), + ) + return nil, fmt.Errorf("ai slot mapper returned invalid json: %w", err) + } + + allowed := make(map[string]struct{}, len(textFields)) + for _, f := range textFields { + allowed[strings.TrimSpace(f.Path)] = struct{}{} + } + out := make([]AISlotAssignment, 0, len(parsed)) + seen := make(map[string]struct{}, len(parsed)) + for _, item := range parsed { + path := strings.TrimSpace(item.FieldPath) + if path == "" { + continue + } + if _, ok := allowed[path]; !ok { + continue + } + if _, dup := seen[path]; dup { + continue + } + slot := strings.TrimSpace(item.Slot) + if !isValidSemanticSlot(slot) { + continue + } + seen[path] = struct{}{} + out = append(out, AISlotAssignment{ + FieldPath: path, + Slot: slot, + Confidence: item.Confidence, + Reason: strings.TrimSpace(item.Reason), + }) + } + sort.SliceStable(out, func(i, j int) bool { return out[i].FieldPath < out[j].FieldPath }) + + mappingLogger().InfoContext(ctx, "ai slot mapping", + "component", "automapper", + "step", "result", + "status", "success", + "provider", provider, + "model", model, + "template_id", templateID, + "input_count", len(textFields), + "mapped_count", len(out), + "duration_ms", time.Since(started).Milliseconds(), + ) + return out, nil +} + +func buildAISlotPrompts(fields []domain.TemplateField) (string, string) { + type fieldPayload struct { + Path string `json:"path"` + KeyName string `json:"keyName"` + Section string `json:"section"` + Block string `json:"block,omitempty"` + DisplayOrder int `json:"displayOrder"` + SampleValue string `json:"sampleValue,omitempty"` + } + payload := make([]fieldPayload, 0, len(fields)) + for _, f := range fields { + payload = append(payload, fieldPayload{ + Path: strings.TrimSpace(f.Path), + KeyName: strings.TrimSpace(f.KeyName), + Section: strings.TrimSpace(f.Section), + Block: semanticExtractBlockID(f), + DisplayOrder: f.DisplayOrder, + SampleValue: truncateSample(f.SampleValue, 200), + }) + } + body := map[string]any{ + "fields": payload, + } + bodyJSON, _ := json.MarshalIndent(body, "", " ") + + system := strings.TrimSpace(` +You are a website-template field mapper. You classify each text field of a QC +website template into ONE semantic slot, or skip it. + +Allowed slots: +- hero.title (one per template, topmost large heading) +- intro.title, intro.description (one each, secondary heading/intro block) +- about.description (one, longer "about us" paragraph) +- cta.text (one, call-to-action button / highlight) +- service_items[N].title (N = 0,1,2... one per service card) +- service_items[N].description +- team_items[N].name +- team_items[N].description +- testimonial_items[N].title +- testimonial_items[N].name +- testimonial_items[N].description + +Rules: +1. Use the section (e.g. "services", "testimonials") as a strong hint. +2. Use the block prefix (e.g. "c8987", "r1865") to group title+description from + the same UI block together. +3. Use displayOrder to decide which block is hero vs intro vs about (lower order = earlier on page). +4. Use sampleValue length: short (<60 chars) = title/name; long (>120 chars) = description. +5. For repeated slots, pair each block to ONE item index; use the same N for + title and description coming from the same block. Index from 0, increasing + by block displayOrder. +6. If you cannot map a field to one of the listed slots, output "" (empty) — never invent slots. +7. Each fieldPath appears exactly once in the output. + +Return ONLY JSON, no markdown, no commentary: +{"mappings":[{"fieldPath":"...","slot":"hero.title","confidence":0.9,"reason":"..."}]} +`) + + user := "Classify the following fields. Respond with JSON only.\n\n" + string(bodyJSON) + return system, user +} + +func parseAISlotResponse(raw string) ([]AISlotAssignment, error) { + content := strings.TrimSpace(raw) + if content == "" { + return nil, fmt.Errorf("empty response") + } + candidates := []string{content} + if fenced := extractFencedJSON(content); fenced != "" { + candidates = append([]string{fenced}, candidates...) + } + if obj := extractJSONObject(content); obj != "" { + candidates = append(candidates, obj) + } + + var firstErr error + for _, candidate := range candidates { + items, err := parseAISlotPayload(candidate) + if err == nil { + return items, nil + } + if firstErr == nil { + firstErr = err + } + } + if firstErr != nil { + return nil, firstErr + } + return nil, fmt.Errorf("response is not a valid mapping json") +} + +func parseAISlotPayload(raw string) ([]AISlotAssignment, error) { + var root any + if err := json.Unmarshal([]byte(raw), &root); err != nil { + return nil, err + } + var itemsRaw []any + switch value := root.(type) { + case map[string]any: + list, ok := value["mappings"].([]any) + if !ok { + // also tolerate {"suggestions":...} or any single array property + for _, key := range []string{"suggestions", "results", "items"} { + if arr, ok := value[key].([]any); ok { + list = arr + break + } + } + } + if list == nil { + return nil, fmt.Errorf("object must contain \"mappings\" array") + } + itemsRaw = list + case []any: + itemsRaw = value + default: + return nil, fmt.Errorf("payload must be object or array") + } + out := make([]AISlotAssignment, 0, len(itemsRaw)) + for idx, item := range itemsRaw { + m, ok := item.(map[string]any) + if !ok { + return nil, fmt.Errorf("entry #%d is not an object", idx+1) + } + path := strings.TrimSpace(anyToString(m["fieldPath"])) + if path == "" { + path = strings.TrimSpace(anyToString(m["path"])) + } + if path == "" { + continue + } + confidence := 0.0 + if v, ok := m["confidence"].(float64); ok { + confidence = v + } + out = append(out, AISlotAssignment{ + FieldPath: path, + Slot: strings.TrimSpace(anyToString(m["slot"])), + Confidence: confidence, + Reason: strings.TrimSpace(anyToString(m["reason"])), + }) + } + return out, nil +} + +func truncateSample(value string, limit int) string { + trimmed := strings.TrimSpace(value) + if trimmed == "" || limit <= 0 { + return "" + } + runes := []rune(trimmed) + if len(runes) <= limit { + return trimmed + } + return string(runes[:limit]) + "..." +} diff --git a/internal/mapping/semantic_slots.go b/internal/mapping/semantic_slots.go index ab16387..96bcb45 100644 --- a/internal/mapping/semantic_slots.go +++ b/internal/mapping/semantic_slots.go @@ -52,6 +52,26 @@ func MapTemplateFieldsToSemanticSlots(fields []domain.TemplateField) SemanticSlo continue } + blockID := semanticExtractBlockID(field) + + // Persisted slot (from AI or manual override) wins over rule-based logic. + if storedSlot := strings.TrimSpace(field.SemanticSlot); storedSlot != "" { + source := strings.TrimSpace(field.MappingSource) + if source == "" { + source = "stored" + } + targets = append(targets, SemanticSlotTarget{ + Slot: storedSlot, + FieldPath: strings.TrimSpace(field.Path), + FieldKey: strings.TrimSpace(field.KeyName), + DisplayLabel: strings.TrimSpace(field.DisplayLabel), + WebsiteSection: semanticSection(field), + BlockID: blockID, + Reason: "source=" + source, + }) + continue + } + section := semanticSection(field) role := semanticRole(field) slot, mapped := semanticSlotForField(field, section, role, sectionGroupIndex, sectionGroupNext, repeatedIndexResolver) @@ -59,8 +79,8 @@ func MapTemplateFieldsToSemanticSlots(fields []domain.TemplateField) SemanticSlo continue } - reason := "section=" + section + ", role=" + role - if blockID := semanticExtractBlockID(field); blockID != "" { + reason := "rule: section=" + section + ", role=" + role + if blockID != "" { reason += ", block=" + blockID } @@ -70,7 +90,7 @@ func MapTemplateFieldsToSemanticSlots(fields []domain.TemplateField) SemanticSlo FieldKey: strings.TrimSpace(field.KeyName), DisplayLabel: strings.TrimSpace(field.DisplayLabel), WebsiteSection: section, - BlockID: semanticExtractBlockID(field), + BlockID: blockID, Reason: reason, }) } diff --git a/internal/onboarding/service.go b/internal/onboarding/service.go index ed02b17..9fd5239 100644 --- a/internal/onboarding/service.go +++ b/internal/onboarding/service.go @@ -4,6 +4,7 @@ import ( "context" "encoding/json" "fmt" + "log/slog" "regexp" "sort" "strconv" @@ -11,6 +12,7 @@ import ( "time" "qctextbuilder/internal/domain" + "qctextbuilder/internal/mapping" "qctextbuilder/internal/qcclient" "qctextbuilder/internal/store" ) @@ -35,6 +37,7 @@ type Service struct { qc qcclient.Client templateStore store.TemplateStore manifestStore store.ManifestStore + aiSlotMapper *mapping.AISlotMapper } type FieldPatch struct { @@ -45,6 +48,7 @@ type FieldPatch struct { DisplayOrder *int WebsiteSection *string Notes *string + SemanticSlot *string } func New(qc qcclient.Client, templateStore store.TemplateStore, manifestStore store.ManifestStore) *Service { @@ -55,6 +59,11 @@ func New(qc qcclient.Client, templateStore store.TemplateStore, manifestStore st } } +func (s *Service) WithAISlotMapper(mapper *mapping.AISlotMapper) *Service { + s.aiSlotMapper = mapper + return s +} + func (s *Service) OnboardTemplate(ctx context.Context, templateID int64) (*domain.TemplateManifest, []domain.TemplateField, error) { template, err := s.templateStore.GetTemplateByID(ctx, templateID) if err != nil { @@ -73,6 +82,7 @@ func (s *Service) OnboardTemplate(ctx context.Context, templateID int64) (*domai manifestID := strconv.FormatInt(time.Now().UnixNano(), 10) now := time.Now().UTC() fields := flattenDiscovery(templateID, manifestID, data) + s.applyAISlotMapping(ctx, templateID, fields) flattened, _ := json.Marshal(fields) reqRaw, _ := json.Marshal(req) @@ -157,6 +167,19 @@ func (s *Service) UpdateTemplateFields(ctx context.Context, templateID int64, ma if patch.Notes != nil { fields[idx].Notes = strings.TrimSpace(*patch.Notes) } + if patch.SemanticSlot != nil { + newSlot := strings.TrimSpace(*patch.SemanticSlot) + if newSlot != fields[idx].SemanticSlot { + fields[idx].SemanticSlot = newSlot + if newSlot == "" { + fields[idx].MappingSource = "" + fields[idx].MappingConfidence = 0 + } else { + fields[idx].MappingSource = domain.MappingSourceManual + fields[idx].MappingConfidence = 1.0 + } + } + } } if err := s.manifestStore.UpdateFields(ctx, manifest.ID, fields); err != nil { @@ -262,6 +285,47 @@ func isLikelyImagePlaceholder(sample string) bool { return strings.Contains(normalized, " image image ") } +// applyAISlotMapping runs the AI slot mapper (if configured) and writes the +// resulting semantic slot + source onto the fields in-place. Failures are +// logged and ignored — onboarding still succeeds, and rule-based mapping will +// take over at build time. +func (s *Service) applyAISlotMapping(ctx context.Context, templateID int64, fields []domain.TemplateField) { + if s == nil || s.aiSlotMapper == nil || len(fields) == 0 { + return + } + assignments, err := s.aiSlotMapper.MapFields(ctx, templateID, fields) + if err != nil { + slog.WarnContext(ctx, "ai slot mapping skipped", + "component", "onboarding", + "template_id", templateID, + "error", err.Error(), + ) + return + } + if len(assignments) == 0 { + return + } + bySlot := make(map[string]mapping.AISlotAssignment, len(assignments)) + for _, a := range assignments { + bySlot[a.FieldPath] = a + } + for i := range fields { + assignment, ok := bySlot[strings.TrimSpace(fields[i].Path)] + if !ok { + continue + } + fields[i].SemanticSlot = assignment.Slot + fields[i].MappingSource = domain.MappingSourceAI + fields[i].MappingConfidence = assignment.Confidence + } + slog.InfoContext(ctx, "ai slot mapping applied", + "component", "onboarding", + "template_id", templateID, + "mapped_count", len(assignments), + "total_fields", len(fields), + ) +} + func isLikelyImagePath(path string) bool { normalized := strings.ToLower(strings.TrimSpace(path)) for _, hint := range imageLikePathHints { diff --git a/internal/store/sqlite/migrations/009_add_semantic_slot_to_fields.sql b/internal/store/sqlite/migrations/009_add_semantic_slot_to_fields.sql new file mode 100644 index 0000000..5f15020 --- /dev/null +++ b/internal/store/sqlite/migrations/009_add_semantic_slot_to_fields.sql @@ -0,0 +1,8 @@ +ALTER TABLE qc_template_fields +ADD COLUMN semantic_slot TEXT NOT NULL DEFAULT ''; + +ALTER TABLE qc_template_fields +ADD COLUMN mapping_source TEXT NOT NULL DEFAULT ''; + +ALTER TABLE qc_template_fields +ADD COLUMN mapping_confidence REAL NOT NULL DEFAULT 0; diff --git a/internal/store/sqlite/store.go b/internal/store/sqlite/store.go index 25c7f48..5c7e25c 100644 --- a/internal/store/sqlite/store.go +++ b/internal/store/sqlite/store.go @@ -189,10 +189,12 @@ func (s *Store) CreateManifest(ctx context.Context, manifest domain.TemplateMani _, err := tx.ExecContext(ctx, ` INSERT INTO qc_template_fields ( id, template_id, manifest_id, section, website_section, key_name, path, field_kind, - sample_value, is_enabled, is_required_by_us, display_label, display_order, notes - ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, + sample_value, is_enabled, is_required_by_us, display_label, display_order, notes, + semantic_slot, mapping_source, mapping_confidence + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, f.ID, f.TemplateID, f.ManifestID, f.Section, domain.NormalizeWebsiteSection(f.WebsiteSection), f.KeyName, f.Path, f.FieldKind, f.SampleValue, boolToInt(f.IsEnabled), boolToInt(f.IsRequiredByUs), f.DisplayLabel, f.DisplayOrder, f.Notes, + strings.TrimSpace(f.SemanticSlot), strings.TrimSpace(f.MappingSource), f.MappingConfidence, ) if err != nil { return err @@ -219,7 +221,8 @@ func (s *Store) GetActiveManifestByTemplateID(ctx context.Context, templateID in func (s *Store) ListFieldsByManifestID(ctx context.Context, manifestID string) ([]domain.TemplateField, error) { rows, err := s.db.QueryContext(ctx, ` SELECT id, template_id, manifest_id, section, website_section, key_name, path, field_kind, sample_value, - is_enabled, is_required_by_us, display_label, display_order, notes + is_enabled, is_required_by_us, display_label, display_order, notes, + semantic_slot, mapping_source, mapping_confidence FROM qc_template_fields WHERE manifest_id = ? ORDER BY display_order ASC, id ASC`, manifestID) @@ -235,12 +238,15 @@ func (s *Store) ListFieldsByManifestID(ctx context.Context, manifestID string) ( if err := rows.Scan( &f.ID, &f.TemplateID, &f.ManifestID, &f.Section, &f.WebsiteSection, &f.KeyName, &f.Path, &f.FieldKind, &f.SampleValue, &isEnabled, &isRequired, &f.DisplayLabel, &f.DisplayOrder, &f.Notes, + &f.SemanticSlot, &f.MappingSource, &f.MappingConfidence, ); err != nil { return nil, err } f.IsEnabled = isEnabled == 1 f.IsRequiredByUs = isRequired == 1 f.WebsiteSection = domain.NormalizeWebsiteSection(f.WebsiteSection) + f.SemanticSlot = strings.TrimSpace(f.SemanticSlot) + f.MappingSource = strings.TrimSpace(f.MappingSource) fields = append(fields, f) } if err := rows.Err(); err != nil { @@ -273,10 +279,12 @@ func (s *Store) UpdateFields(ctx context.Context, manifestID string, fields []do _, err := tx.ExecContext(ctx, ` INSERT INTO qc_template_fields ( id, template_id, manifest_id, section, website_section, key_name, path, field_kind, - sample_value, is_enabled, is_required_by_us, display_label, display_order, notes - ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, + sample_value, is_enabled, is_required_by_us, display_label, display_order, notes, + semantic_slot, mapping_source, mapping_confidence + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, f.ID, f.TemplateID, f.ManifestID, f.Section, domain.NormalizeWebsiteSection(f.WebsiteSection), f.KeyName, f.Path, f.FieldKind, f.SampleValue, boolToInt(f.IsEnabled), boolToInt(f.IsRequiredByUs), f.DisplayLabel, f.DisplayOrder, f.Notes, + strings.TrimSpace(f.SemanticSlot), strings.TrimSpace(f.MappingSource), f.MappingConfidence, ) if err != nil { return err diff --git a/web/templates/template_detail.gohtml b/web/templates/template_detail.gohtml index 0e9ff29..ec6afe8 100644 --- a/web/templates/template_detail.gohtml +++ b/web/templates/template_detail.gohtml @@ -51,6 +51,7 @@ Label Order Website Section + Semantic Slot Notes Sample @@ -74,6 +75,17 @@ {{end}} + + + {{if $f.MappingSource}}{{$f.MappingSource}}{{end}} + {{$f.SampleValue}}