- ManifestField bekommt semantic_slot/mapping_source/mapping_confidence (Migration 009, Domain-Model, SQLite-Store). - Neuer AISlotMapper ruft den aktiven LLM-Provider mit cleverem Prompt: klassifiziert jedes Textfeld in einen festen Slot-Katalog (hero/intro/ about/cta/service_items[N]/team_items[N]/testimonial_items[N]) anhand Section, Block-ID, displayOrder und Sample-Wert. - Onboarding ruft den Mapper nach Discovery auf und persistiert die Slots als source=ai. Fehler werden gelogged, Onboarding bleibt erfolgreich. - MapTemplateFieldsToSemanticSlots respektiert persistierte Slots; die Rule-basierte Logik laeuft nur fuer Felder ohne Slot weiter. - Template-Detail-UI hat neue Dropdown-Spalte "Semantic Slot"; manueller Override setzt source=manual. Custom-Werte (ausserhalb der Liste) bleiben sichtbar und werden nicht versehentlich ueberschrieben. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>master
| @@ -71,11 +71,12 @@ func New(cfg config.Config) (*App, error) { | |||||
| qc := qcclient.New(cfg.QCBaseURL, cfg.QCToken, 15*time.Second, logger) | qc := qcclient.New(cfg.QCBaseURL, cfg.QCToken, 15*time.Second, logger) | ||||
| templateSvc := templatesvc.New(qc, templateStore, manifestStore) | templateSvc := templatesvc.New(qc, templateStore, manifestStore) | ||||
| onboardSvc := onboarding.New(qc, templateStore, manifestStore) | |||||
| draftSvc := draftsvc.New(draftStore, templateStore, manifestStore) | draftSvc := draftsvc.New(draftStore, templateStore, manifestStore) | ||||
| mappingSvc := mapping.New() | mappingSvc := mapping.New() | ||||
| buildSvc := buildsvc.New(qc, templateStore, manifestStore, buildStore, mappingSvc, time.Duration(cfg.PollTimeoutSeconds)*time.Second) | buildSvc := buildsvc.New(qc, templateStore, manifestStore, buildStore, mappingSvc, time.Duration(cfg.PollTimeoutSeconds)*time.Second) | ||||
| providerRuntime := llmruntime.NewFactory(45 * time.Second) | providerRuntime := llmruntime.NewFactory(45 * time.Second) | ||||
| onboardSvc := onboarding.New(qc, templateStore, manifestStore). | |||||
| WithAISlotMapper(mapping.NewAISlotMapper(settingsStore, providerRuntime)) | |||||
| suggestionGenerator := mapping.NewCompositeSuggestionGenerator( | suggestionGenerator := mapping.NewCompositeSuggestionGenerator( | ||||
| mapping.NewGooglePlacesSuggestionGenerator(settingsStore), | mapping.NewGooglePlacesSuggestionGenerator(settingsStore), | ||||
| mapping.NewCompositeSuggestionGenerator( | mapping.NewCompositeSuggestionGenerator( | ||||
| @@ -36,22 +36,31 @@ type TemplateManifest struct { | |||||
| } | } | ||||
| type TemplateField struct { | type TemplateField struct { | ||||
| ID string `json:"id"` | |||||
| TemplateID int64 `json:"templateId"` | |||||
| ManifestID string `json:"manifestId"` | |||||
| Section string `json:"section"` | |||||
| WebsiteSection string `json:"websiteSection"` | |||||
| KeyName string `json:"keyName"` | |||||
| Path string `json:"path"` | |||||
| FieldKind string `json:"fieldKind"` | |||||
| SampleValue string `json:"sampleValue"` | |||||
| IsEnabled bool `json:"isEnabled"` | |||||
| IsRequiredByUs bool `json:"isRequiredByUs"` | |||||
| DisplayLabel string `json:"displayLabel"` | |||||
| DisplayOrder int `json:"displayOrder"` | |||||
| Notes string `json:"notes"` | |||||
| ID string `json:"id"` | |||||
| TemplateID int64 `json:"templateId"` | |||||
| ManifestID string `json:"manifestId"` | |||||
| Section string `json:"section"` | |||||
| WebsiteSection string `json:"websiteSection"` | |||||
| KeyName string `json:"keyName"` | |||||
| Path string `json:"path"` | |||||
| FieldKind string `json:"fieldKind"` | |||||
| SampleValue string `json:"sampleValue"` | |||||
| IsEnabled bool `json:"isEnabled"` | |||||
| IsRequiredByUs bool `json:"isRequiredByUs"` | |||||
| DisplayLabel string `json:"displayLabel"` | |||||
| DisplayOrder int `json:"displayOrder"` | |||||
| Notes string `json:"notes"` | |||||
| SemanticSlot string `json:"semanticSlot,omitempty"` | |||||
| MappingSource string `json:"mappingSource,omitempty"` | |||||
| MappingConfidence float64 `json:"mappingConfidence,omitempty"` | |||||
| } | } | ||||
| const ( | |||||
| MappingSourceRule = "rule" | |||||
| MappingSourceAI = "ai" | |||||
| MappingSourceManual = "manual" | |||||
| ) | |||||
| type SiteBuild struct { | type SiteBuild struct { | ||||
| ID string `json:"id"` | ID string `json:"id"` | ||||
| TemplateID int64 `json:"templateId"` | TemplateID int64 `json:"templateId"` | ||||
| @@ -86,15 +86,19 @@ type templatesPageData struct { | |||||
| } | } | ||||
| type templateFieldView struct { | type templateFieldView struct { | ||||
| Path string | |||||
| FieldKind string | |||||
| IsEnabled bool | |||||
| IsRequiredByUs bool | |||||
| DisplayLabel string | |||||
| DisplayOrder int | |||||
| WebsiteSection string | |||||
| Notes string | |||||
| SampleValue string | |||||
| Path string | |||||
| FieldKind string | |||||
| IsEnabled bool | |||||
| IsRequiredByUs bool | |||||
| DisplayLabel string | |||||
| DisplayOrder int | |||||
| WebsiteSection string | |||||
| Notes string | |||||
| SampleValue string | |||||
| SemanticSlot string | |||||
| SemanticSlotIsKnown bool | |||||
| MappingSource string | |||||
| MappingConfidence float64 | |||||
| } | } | ||||
| type websiteSectionOptionView struct { | type websiteSectionOptionView struct { | ||||
| @@ -102,11 +106,17 @@ type websiteSectionOptionView struct { | |||||
| Label string | Label string | ||||
| } | } | ||||
| type semanticSlotOptionView struct { | |||||
| Value string | |||||
| Label string | |||||
| } | |||||
| type templateDetailPageData struct { | type templateDetailPageData struct { | ||||
| pageData | pageData | ||||
| Detail *templatesvc.TemplateDetail | Detail *templatesvc.TemplateDetail | ||||
| Fields []templateFieldView | Fields []templateFieldView | ||||
| WebsiteSectionOptions []websiteSectionOptionView | WebsiteSectionOptions []websiteSectionOptionView | ||||
| SemanticSlotOptions []semanticSlotOptionView | |||||
| } | } | ||||
| type buildFieldView struct { | type buildFieldView struct { | ||||
| @@ -359,18 +369,28 @@ func (u *UI) TemplateDetail(w http.ResponseWriter, r *http.Request) { | |||||
| http.Error(w, err.Error(), http.StatusNotFound) | http.Error(w, err.Error(), http.StatusNotFound) | ||||
| return | return | ||||
| } | } | ||||
| slotOpts := semanticSlotOptions() | |||||
| knownSlots := make(map[string]bool, len(slotOpts)) | |||||
| for _, opt := range slotOpts { | |||||
| knownSlots[opt.Value] = true | |||||
| } | |||||
| fields := make([]templateFieldView, 0, len(detail.Fields)) | fields := make([]templateFieldView, 0, len(detail.Fields)) | ||||
| for _, f := range detail.Fields { | for _, f := range detail.Fields { | ||||
| slot := strings.TrimSpace(f.SemanticSlot) | |||||
| fields = append(fields, templateFieldView{ | fields = append(fields, templateFieldView{ | ||||
| Path: f.Path, | |||||
| FieldKind: f.FieldKind, | |||||
| IsEnabled: f.IsEnabled, | |||||
| IsRequiredByUs: f.IsRequiredByUs, | |||||
| DisplayLabel: f.DisplayLabel, | |||||
| DisplayOrder: f.DisplayOrder, | |||||
| WebsiteSection: domain.NormalizeWebsiteSection(f.WebsiteSection), | |||||
| Notes: f.Notes, | |||||
| SampleValue: f.SampleValue, | |||||
| Path: f.Path, | |||||
| FieldKind: f.FieldKind, | |||||
| IsEnabled: f.IsEnabled, | |||||
| IsRequiredByUs: f.IsRequiredByUs, | |||||
| DisplayLabel: f.DisplayLabel, | |||||
| DisplayOrder: f.DisplayOrder, | |||||
| WebsiteSection: domain.NormalizeWebsiteSection(f.WebsiteSection), | |||||
| Notes: f.Notes, | |||||
| SampleValue: f.SampleValue, | |||||
| SemanticSlot: slot, | |||||
| SemanticSlotIsKnown: slot == "" || knownSlots[slot], | |||||
| MappingSource: f.MappingSource, | |||||
| MappingConfidence: f.MappingConfidence, | |||||
| }) | }) | ||||
| } | } | ||||
| u.render.Render(w, "template_detail", templateDetailPageData{ | u.render.Render(w, "template_detail", templateDetailPageData{ | ||||
| @@ -378,6 +398,7 @@ func (u *UI) TemplateDetail(w http.ResponseWriter, r *http.Request) { | |||||
| Detail: detail, | Detail: detail, | ||||
| Fields: fields, | Fields: fields, | ||||
| WebsiteSectionOptions: websiteSectionOptions(), | WebsiteSectionOptions: websiteSectionOptions(), | ||||
| SemanticSlotOptions: slotOpts, | |||||
| }) | }) | ||||
| } | } | ||||
| @@ -420,6 +441,7 @@ func (u *UI) UpdateTemplateFields(w http.ResponseWriter, r *http.Request) { | |||||
| http.Redirect(w, r, fmt.Sprintf("/templates/%d?err=%s", templateID, urlQuery("invalid display order")), http.StatusSeeOther) | http.Redirect(w, r, fmt.Sprintf("/templates/%d?err=%s", templateID, urlQuery("invalid display order")), http.StatusSeeOther) | ||||
| return | return | ||||
| } | } | ||||
| semanticSlot := strings.TrimSpace(r.FormValue(fmt.Sprintf("field_semantic_slot_%d", i))) | |||||
| patches = append(patches, onboarding.FieldPatch{ | patches = append(patches, onboarding.FieldPatch{ | ||||
| Path: path, | Path: path, | ||||
| IsEnabled: boolPtr(enabled), | IsEnabled: boolPtr(enabled), | ||||
| @@ -428,6 +450,7 @@ func (u *UI) UpdateTemplateFields(w http.ResponseWriter, r *http.Request) { | |||||
| DisplayOrder: intPtr(order), | DisplayOrder: intPtr(order), | ||||
| WebsiteSection: strPtr(websiteSection), | WebsiteSection: strPtr(websiteSection), | ||||
| Notes: strPtr(notes), | Notes: strPtr(notes), | ||||
| SemanticSlot: strPtr(semanticSlot), | |||||
| }) | }) | ||||
| } | } | ||||
| @@ -2152,3 +2175,41 @@ func websiteSectionOptions() []websiteSectionOptionView { | |||||
| } | } | ||||
| return out | return out | ||||
| } | } | ||||
| func semanticSlotOptions() []semanticSlotOptionView { | |||||
| // "" = automatisch (rule-basiertes Fallback). Reihenfolge ist UI-relevant. | |||||
| slots := []semanticSlotOptionView{ | |||||
| {Value: "", Label: "(automatisch)"}, | |||||
| {Value: "hero.title", Label: "Hero - Title"}, | |||||
| {Value: "intro.title", Label: "Intro - Title"}, | |||||
| {Value: "intro.description", Label: "Intro - Description"}, | |||||
| {Value: "about.description", Label: "About - Description"}, | |||||
| {Value: "cta.text", Label: "CTA - Text"}, | |||||
| {Value: "service_items[0].title", Label: "Service Item #1 - Title"}, | |||||
| {Value: "service_items[0].description", Label: "Service Item #1 - Description"}, | |||||
| {Value: "service_items[1].title", Label: "Service Item #2 - Title"}, | |||||
| {Value: "service_items[1].description", Label: "Service Item #2 - Description"}, | |||||
| {Value: "service_items[2].title", Label: "Service Item #3 - Title"}, | |||||
| {Value: "service_items[2].description", Label: "Service Item #3 - Description"}, | |||||
| {Value: "service_items[3].title", Label: "Service Item #4 - Title"}, | |||||
| {Value: "service_items[3].description", Label: "Service Item #4 - Description"}, | |||||
| {Value: "service_items[4].title", Label: "Service Item #5 - Title"}, | |||||
| {Value: "service_items[4].description", Label: "Service Item #5 - Description"}, | |||||
| {Value: "team_items[0].name", Label: "Team #1 - Name"}, | |||||
| {Value: "team_items[0].description", Label: "Team #1 - Description"}, | |||||
| {Value: "team_items[1].name", Label: "Team #2 - Name"}, | |||||
| {Value: "team_items[1].description", Label: "Team #2 - Description"}, | |||||
| {Value: "team_items[2].name", Label: "Team #3 - Name"}, | |||||
| {Value: "team_items[2].description", Label: "Team #3 - Description"}, | |||||
| {Value: "testimonial_items[0].title", Label: "Testimonial #1 - Title"}, | |||||
| {Value: "testimonial_items[0].name", Label: "Testimonial #1 - Name"}, | |||||
| {Value: "testimonial_items[0].description", Label: "Testimonial #1 - Description"}, | |||||
| {Value: "testimonial_items[1].title", Label: "Testimonial #2 - Title"}, | |||||
| {Value: "testimonial_items[1].name", Label: "Testimonial #2 - Name"}, | |||||
| {Value: "testimonial_items[1].description", Label: "Testimonial #2 - Description"}, | |||||
| {Value: "testimonial_items[2].title", Label: "Testimonial #3 - Title"}, | |||||
| {Value: "testimonial_items[2].name", Label: "Testimonial #3 - Name"}, | |||||
| {Value: "testimonial_items[2].description", Label: "Testimonial #3 - Description"}, | |||||
| } | |||||
| return slots | |||||
| } | |||||
| @@ -0,0 +1,362 @@ | |||||
| package mapping | |||||
| import ( | |||||
| "context" | |||||
| "encoding/json" | |||||
| "fmt" | |||||
| "regexp" | |||||
| "sort" | |||||
| "strings" | |||||
| "time" | |||||
| "qctextbuilder/internal/domain" | |||||
| "qctextbuilder/internal/llmruntime" | |||||
| ) | |||||
| // AISlotMapper classifies template fields into semantic slots via the active | |||||
| // LLM provider. It is meant to run once per template (during onboarding) and | |||||
| // its results are persisted on the manifest fields. | |||||
| type AISlotMapper struct { | |||||
| settings SettingsReader | |||||
| runtimeFactory *llmruntime.Factory | |||||
| } | |||||
| func NewAISlotMapper(settings SettingsReader, runtimeFactory *llmruntime.Factory) *AISlotMapper { | |||||
| return &AISlotMapper{settings: settings, runtimeFactory: runtimeFactory} | |||||
| } | |||||
| type AISlotAssignment struct { | |||||
| FieldPath string `json:"fieldPath"` | |||||
| Slot string `json:"slot"` | |||||
| Confidence float64 `json:"confidence"` | |||||
| Reason string `json:"reason"` | |||||
| } | |||||
| var aiSlotRepeatedPattern = regexp.MustCompile(`^(service_items|team_items|testimonial_items)\[\d+\]\.(title|description|name)$`) | |||||
| var aiSlotSingletons = map[string]struct{}{ | |||||
| "hero.title": {}, | |||||
| "intro.title": {}, | |||||
| "intro.description": {}, | |||||
| "about.description": {}, | |||||
| "cta.text": {}, | |||||
| } | |||||
| func isValidSemanticSlot(slot string) bool { | |||||
| if slot == "" { | |||||
| return false | |||||
| } | |||||
| if _, ok := aiSlotSingletons[slot]; ok { | |||||
| return true | |||||
| } | |||||
| if !aiSlotRepeatedPattern.MatchString(slot) { | |||||
| return false | |||||
| } | |||||
| // team_items[N].title is not a valid combination — name is the only label role. | |||||
| switch { | |||||
| case strings.HasPrefix(slot, "team_items[") && strings.HasSuffix(slot, ".title"): | |||||
| return false | |||||
| case strings.HasPrefix(slot, "service_items[") && strings.HasSuffix(slot, ".name"): | |||||
| return false | |||||
| } | |||||
| return true | |||||
| } | |||||
| // MapFields returns one assignment per input field whose Slot is a valid | |||||
| // semantic slot. Fields the model decided to skip are omitted. | |||||
| func (m *AISlotMapper) MapFields(ctx context.Context, templateID int64, fields []domain.TemplateField) ([]AISlotAssignment, error) { | |||||
| started := time.Now() | |||||
| if m == nil || m.settings == nil || m.runtimeFactory == nil { | |||||
| return nil, fmt.Errorf("ai slot mapper is not configured") | |||||
| } | |||||
| textFields := make([]domain.TemplateField, 0, len(fields)) | |||||
| for _, f := range fields { | |||||
| if !f.IsEnabled { | |||||
| continue | |||||
| } | |||||
| if !strings.EqualFold(strings.TrimSpace(f.FieldKind), "text") { | |||||
| continue | |||||
| } | |||||
| textFields = append(textFields, f) | |||||
| } | |||||
| if len(textFields) == 0 { | |||||
| return []AISlotAssignment{}, nil | |||||
| } | |||||
| settings, err := m.settings.GetSettings(ctx) | |||||
| if err != nil || settings == nil { | |||||
| return nil, fmt.Errorf("llm settings are not available") | |||||
| } | |||||
| provider := domain.NormalizeLLMProvider(settings.LLMActiveProvider) | |||||
| model := domain.NormalizeLLMModel(provider, settings.LLMActiveModel) | |||||
| if strings.TrimSpace(model) == "" { | |||||
| return nil, fmt.Errorf("no active llm model configured") | |||||
| } | |||||
| apiKey := domain.LLMAPIKeyForProvider(provider, *settings) | |||||
| if provider != domain.LLMProviderOllama && strings.TrimSpace(apiKey) == "" { | |||||
| return nil, fmt.Errorf("api key for provider %s is not configured", provider) | |||||
| } | |||||
| baseURL := strings.TrimSpace(settings.LLMBaseURL) | |||||
| client, err := m.runtimeFactory.ClientFor(provider) | |||||
| if err != nil { | |||||
| return nil, err | |||||
| } | |||||
| systemPrompt, userPrompt := buildAISlotPrompts(textFields) | |||||
| mappingLogger().InfoContext(ctx, "ai slot mapping", | |||||
| "component", "automapper", | |||||
| "step", "request", | |||||
| "status", "start", | |||||
| "provider", provider, | |||||
| "model", model, | |||||
| "template_id", templateID, | |||||
| "field_count", len(textFields), | |||||
| ) | |||||
| temperature := 0.0 | |||||
| maxTokens := domain.NormalizeLLMMaxTokens(settings.LLMMaxTokens) | |||||
| if maxTokens < 2048 { | |||||
| maxTokens = 2048 | |||||
| } | |||||
| raw, err := client.Generate(ctx, llmruntime.Request{ | |||||
| Provider: provider, | |||||
| Model: model, | |||||
| BaseURL: baseURL, | |||||
| APIKey: strings.TrimSpace(apiKey), | |||||
| Temperature: &temperature, | |||||
| MaxTokens: &maxTokens, | |||||
| SystemPrompt: systemPrompt, | |||||
| UserPrompt: userPrompt, | |||||
| }) | |||||
| if err != nil { | |||||
| mappingLogger().WarnContext(ctx, "ai slot mapping", | |||||
| "component", "automapper", | |||||
| "step", "request", | |||||
| "status", "failed", | |||||
| "provider", provider, | |||||
| "model", model, | |||||
| "template_id", templateID, | |||||
| "error", shortErr(err), | |||||
| "duration_ms", time.Since(started).Milliseconds(), | |||||
| ) | |||||
| return nil, fmt.Errorf("ai slot mapper request failed (provider=%s model=%s): %w", provider, model, err) | |||||
| } | |||||
| parsed, err := parseAISlotResponse(raw) | |||||
| if err != nil { | |||||
| mappingLogger().WarnContext(ctx, "ai slot mapping", | |||||
| "component", "automapper", | |||||
| "step", "parse", | |||||
| "status", "failed", | |||||
| "provider", provider, | |||||
| "model", model, | |||||
| "template_id", templateID, | |||||
| "error", shortErr(err), | |||||
| "response_snippet", providerLogSnippet(raw, 1500), | |||||
| "duration_ms", time.Since(started).Milliseconds(), | |||||
| ) | |||||
| return nil, fmt.Errorf("ai slot mapper returned invalid json: %w", err) | |||||
| } | |||||
| allowed := make(map[string]struct{}, len(textFields)) | |||||
| for _, f := range textFields { | |||||
| allowed[strings.TrimSpace(f.Path)] = struct{}{} | |||||
| } | |||||
| out := make([]AISlotAssignment, 0, len(parsed)) | |||||
| seen := make(map[string]struct{}, len(parsed)) | |||||
| for _, item := range parsed { | |||||
| path := strings.TrimSpace(item.FieldPath) | |||||
| if path == "" { | |||||
| continue | |||||
| } | |||||
| if _, ok := allowed[path]; !ok { | |||||
| continue | |||||
| } | |||||
| if _, dup := seen[path]; dup { | |||||
| continue | |||||
| } | |||||
| slot := strings.TrimSpace(item.Slot) | |||||
| if !isValidSemanticSlot(slot) { | |||||
| continue | |||||
| } | |||||
| seen[path] = struct{}{} | |||||
| out = append(out, AISlotAssignment{ | |||||
| FieldPath: path, | |||||
| Slot: slot, | |||||
| Confidence: item.Confidence, | |||||
| Reason: strings.TrimSpace(item.Reason), | |||||
| }) | |||||
| } | |||||
| sort.SliceStable(out, func(i, j int) bool { return out[i].FieldPath < out[j].FieldPath }) | |||||
| mappingLogger().InfoContext(ctx, "ai slot mapping", | |||||
| "component", "automapper", | |||||
| "step", "result", | |||||
| "status", "success", | |||||
| "provider", provider, | |||||
| "model", model, | |||||
| "template_id", templateID, | |||||
| "input_count", len(textFields), | |||||
| "mapped_count", len(out), | |||||
| "duration_ms", time.Since(started).Milliseconds(), | |||||
| ) | |||||
| return out, nil | |||||
| } | |||||
| func buildAISlotPrompts(fields []domain.TemplateField) (string, string) { | |||||
| type fieldPayload struct { | |||||
| Path string `json:"path"` | |||||
| KeyName string `json:"keyName"` | |||||
| Section string `json:"section"` | |||||
| Block string `json:"block,omitempty"` | |||||
| DisplayOrder int `json:"displayOrder"` | |||||
| SampleValue string `json:"sampleValue,omitempty"` | |||||
| } | |||||
| payload := make([]fieldPayload, 0, len(fields)) | |||||
| for _, f := range fields { | |||||
| payload = append(payload, fieldPayload{ | |||||
| Path: strings.TrimSpace(f.Path), | |||||
| KeyName: strings.TrimSpace(f.KeyName), | |||||
| Section: strings.TrimSpace(f.Section), | |||||
| Block: semanticExtractBlockID(f), | |||||
| DisplayOrder: f.DisplayOrder, | |||||
| SampleValue: truncateSample(f.SampleValue, 200), | |||||
| }) | |||||
| } | |||||
| body := map[string]any{ | |||||
| "fields": payload, | |||||
| } | |||||
| bodyJSON, _ := json.MarshalIndent(body, "", " ") | |||||
| system := strings.TrimSpace(` | |||||
| You are a website-template field mapper. You classify each text field of a QC | |||||
| website template into ONE semantic slot, or skip it. | |||||
| Allowed slots: | |||||
| - hero.title (one per template, topmost large heading) | |||||
| - intro.title, intro.description (one each, secondary heading/intro block) | |||||
| - about.description (one, longer "about us" paragraph) | |||||
| - cta.text (one, call-to-action button / highlight) | |||||
| - service_items[N].title (N = 0,1,2... one per service card) | |||||
| - service_items[N].description | |||||
| - team_items[N].name | |||||
| - team_items[N].description | |||||
| - testimonial_items[N].title | |||||
| - testimonial_items[N].name | |||||
| - testimonial_items[N].description | |||||
| Rules: | |||||
| 1. Use the section (e.g. "services", "testimonials") as a strong hint. | |||||
| 2. Use the block prefix (e.g. "c8987", "r1865") to group title+description from | |||||
| the same UI block together. | |||||
| 3. Use displayOrder to decide which block is hero vs intro vs about (lower order = earlier on page). | |||||
| 4. Use sampleValue length: short (<60 chars) = title/name; long (>120 chars) = description. | |||||
| 5. For repeated slots, pair each block to ONE item index; use the same N for | |||||
| title and description coming from the same block. Index from 0, increasing | |||||
| by block displayOrder. | |||||
| 6. If you cannot map a field to one of the listed slots, output "" (empty) — never invent slots. | |||||
| 7. Each fieldPath appears exactly once in the output. | |||||
| Return ONLY JSON, no markdown, no commentary: | |||||
| {"mappings":[{"fieldPath":"...","slot":"hero.title","confidence":0.9,"reason":"..."}]} | |||||
| `) | |||||
| user := "Classify the following fields. Respond with JSON only.\n\n" + string(bodyJSON) | |||||
| return system, user | |||||
| } | |||||
| func parseAISlotResponse(raw string) ([]AISlotAssignment, error) { | |||||
| content := strings.TrimSpace(raw) | |||||
| if content == "" { | |||||
| return nil, fmt.Errorf("empty response") | |||||
| } | |||||
| candidates := []string{content} | |||||
| if fenced := extractFencedJSON(content); fenced != "" { | |||||
| candidates = append([]string{fenced}, candidates...) | |||||
| } | |||||
| if obj := extractJSONObject(content); obj != "" { | |||||
| candidates = append(candidates, obj) | |||||
| } | |||||
| var firstErr error | |||||
| for _, candidate := range candidates { | |||||
| items, err := parseAISlotPayload(candidate) | |||||
| if err == nil { | |||||
| return items, nil | |||||
| } | |||||
| if firstErr == nil { | |||||
| firstErr = err | |||||
| } | |||||
| } | |||||
| if firstErr != nil { | |||||
| return nil, firstErr | |||||
| } | |||||
| return nil, fmt.Errorf("response is not a valid mapping json") | |||||
| } | |||||
| func parseAISlotPayload(raw string) ([]AISlotAssignment, error) { | |||||
| var root any | |||||
| if err := json.Unmarshal([]byte(raw), &root); err != nil { | |||||
| return nil, err | |||||
| } | |||||
| var itemsRaw []any | |||||
| switch value := root.(type) { | |||||
| case map[string]any: | |||||
| list, ok := value["mappings"].([]any) | |||||
| if !ok { | |||||
| // also tolerate {"suggestions":...} or any single array property | |||||
| for _, key := range []string{"suggestions", "results", "items"} { | |||||
| if arr, ok := value[key].([]any); ok { | |||||
| list = arr | |||||
| break | |||||
| } | |||||
| } | |||||
| } | |||||
| if list == nil { | |||||
| return nil, fmt.Errorf("object must contain \"mappings\" array") | |||||
| } | |||||
| itemsRaw = list | |||||
| case []any: | |||||
| itemsRaw = value | |||||
| default: | |||||
| return nil, fmt.Errorf("payload must be object or array") | |||||
| } | |||||
| out := make([]AISlotAssignment, 0, len(itemsRaw)) | |||||
| for idx, item := range itemsRaw { | |||||
| m, ok := item.(map[string]any) | |||||
| if !ok { | |||||
| return nil, fmt.Errorf("entry #%d is not an object", idx+1) | |||||
| } | |||||
| path := strings.TrimSpace(anyToString(m["fieldPath"])) | |||||
| if path == "" { | |||||
| path = strings.TrimSpace(anyToString(m["path"])) | |||||
| } | |||||
| if path == "" { | |||||
| continue | |||||
| } | |||||
| confidence := 0.0 | |||||
| if v, ok := m["confidence"].(float64); ok { | |||||
| confidence = v | |||||
| } | |||||
| out = append(out, AISlotAssignment{ | |||||
| FieldPath: path, | |||||
| Slot: strings.TrimSpace(anyToString(m["slot"])), | |||||
| Confidence: confidence, | |||||
| Reason: strings.TrimSpace(anyToString(m["reason"])), | |||||
| }) | |||||
| } | |||||
| return out, nil | |||||
| } | |||||
| func truncateSample(value string, limit int) string { | |||||
| trimmed := strings.TrimSpace(value) | |||||
| if trimmed == "" || limit <= 0 { | |||||
| return "" | |||||
| } | |||||
| runes := []rune(trimmed) | |||||
| if len(runes) <= limit { | |||||
| return trimmed | |||||
| } | |||||
| return string(runes[:limit]) + "..." | |||||
| } | |||||
| @@ -52,6 +52,26 @@ func MapTemplateFieldsToSemanticSlots(fields []domain.TemplateField) SemanticSlo | |||||
| continue | continue | ||||
| } | } | ||||
| blockID := semanticExtractBlockID(field) | |||||
| // Persisted slot (from AI or manual override) wins over rule-based logic. | |||||
| if storedSlot := strings.TrimSpace(field.SemanticSlot); storedSlot != "" { | |||||
| source := strings.TrimSpace(field.MappingSource) | |||||
| if source == "" { | |||||
| source = "stored" | |||||
| } | |||||
| targets = append(targets, SemanticSlotTarget{ | |||||
| Slot: storedSlot, | |||||
| FieldPath: strings.TrimSpace(field.Path), | |||||
| FieldKey: strings.TrimSpace(field.KeyName), | |||||
| DisplayLabel: strings.TrimSpace(field.DisplayLabel), | |||||
| WebsiteSection: semanticSection(field), | |||||
| BlockID: blockID, | |||||
| Reason: "source=" + source, | |||||
| }) | |||||
| continue | |||||
| } | |||||
| section := semanticSection(field) | section := semanticSection(field) | ||||
| role := semanticRole(field) | role := semanticRole(field) | ||||
| slot, mapped := semanticSlotForField(field, section, role, sectionGroupIndex, sectionGroupNext, repeatedIndexResolver) | slot, mapped := semanticSlotForField(field, section, role, sectionGroupIndex, sectionGroupNext, repeatedIndexResolver) | ||||
| @@ -59,8 +79,8 @@ func MapTemplateFieldsToSemanticSlots(fields []domain.TemplateField) SemanticSlo | |||||
| continue | continue | ||||
| } | } | ||||
| reason := "section=" + section + ", role=" + role | |||||
| if blockID := semanticExtractBlockID(field); blockID != "" { | |||||
| reason := "rule: section=" + section + ", role=" + role | |||||
| if blockID != "" { | |||||
| reason += ", block=" + blockID | reason += ", block=" + blockID | ||||
| } | } | ||||
| @@ -70,7 +90,7 @@ func MapTemplateFieldsToSemanticSlots(fields []domain.TemplateField) SemanticSlo | |||||
| FieldKey: strings.TrimSpace(field.KeyName), | FieldKey: strings.TrimSpace(field.KeyName), | ||||
| DisplayLabel: strings.TrimSpace(field.DisplayLabel), | DisplayLabel: strings.TrimSpace(field.DisplayLabel), | ||||
| WebsiteSection: section, | WebsiteSection: section, | ||||
| BlockID: semanticExtractBlockID(field), | |||||
| BlockID: blockID, | |||||
| Reason: reason, | Reason: reason, | ||||
| }) | }) | ||||
| } | } | ||||
| @@ -4,6 +4,7 @@ import ( | |||||
| "context" | "context" | ||||
| "encoding/json" | "encoding/json" | ||||
| "fmt" | "fmt" | ||||
| "log/slog" | |||||
| "regexp" | "regexp" | ||||
| "sort" | "sort" | ||||
| "strconv" | "strconv" | ||||
| @@ -11,6 +12,7 @@ import ( | |||||
| "time" | "time" | ||||
| "qctextbuilder/internal/domain" | "qctextbuilder/internal/domain" | ||||
| "qctextbuilder/internal/mapping" | |||||
| "qctextbuilder/internal/qcclient" | "qctextbuilder/internal/qcclient" | ||||
| "qctextbuilder/internal/store" | "qctextbuilder/internal/store" | ||||
| ) | ) | ||||
| @@ -35,6 +37,7 @@ type Service struct { | |||||
| qc qcclient.Client | qc qcclient.Client | ||||
| templateStore store.TemplateStore | templateStore store.TemplateStore | ||||
| manifestStore store.ManifestStore | manifestStore store.ManifestStore | ||||
| aiSlotMapper *mapping.AISlotMapper | |||||
| } | } | ||||
| type FieldPatch struct { | type FieldPatch struct { | ||||
| @@ -45,6 +48,7 @@ type FieldPatch struct { | |||||
| DisplayOrder *int | DisplayOrder *int | ||||
| WebsiteSection *string | WebsiteSection *string | ||||
| Notes *string | Notes *string | ||||
| SemanticSlot *string | |||||
| } | } | ||||
| func New(qc qcclient.Client, templateStore store.TemplateStore, manifestStore store.ManifestStore) *Service { | func New(qc qcclient.Client, templateStore store.TemplateStore, manifestStore store.ManifestStore) *Service { | ||||
| @@ -55,6 +59,11 @@ func New(qc qcclient.Client, templateStore store.TemplateStore, manifestStore st | |||||
| } | } | ||||
| } | } | ||||
| func (s *Service) WithAISlotMapper(mapper *mapping.AISlotMapper) *Service { | |||||
| s.aiSlotMapper = mapper | |||||
| return s | |||||
| } | |||||
| func (s *Service) OnboardTemplate(ctx context.Context, templateID int64) (*domain.TemplateManifest, []domain.TemplateField, error) { | func (s *Service) OnboardTemplate(ctx context.Context, templateID int64) (*domain.TemplateManifest, []domain.TemplateField, error) { | ||||
| template, err := s.templateStore.GetTemplateByID(ctx, templateID) | template, err := s.templateStore.GetTemplateByID(ctx, templateID) | ||||
| if err != nil { | if err != nil { | ||||
| @@ -73,6 +82,7 @@ func (s *Service) OnboardTemplate(ctx context.Context, templateID int64) (*domai | |||||
| manifestID := strconv.FormatInt(time.Now().UnixNano(), 10) | manifestID := strconv.FormatInt(time.Now().UnixNano(), 10) | ||||
| now := time.Now().UTC() | now := time.Now().UTC() | ||||
| fields := flattenDiscovery(templateID, manifestID, data) | fields := flattenDiscovery(templateID, manifestID, data) | ||||
| s.applyAISlotMapping(ctx, templateID, fields) | |||||
| flattened, _ := json.Marshal(fields) | flattened, _ := json.Marshal(fields) | ||||
| reqRaw, _ := json.Marshal(req) | reqRaw, _ := json.Marshal(req) | ||||
| @@ -157,6 +167,19 @@ func (s *Service) UpdateTemplateFields(ctx context.Context, templateID int64, ma | |||||
| if patch.Notes != nil { | if patch.Notes != nil { | ||||
| fields[idx].Notes = strings.TrimSpace(*patch.Notes) | fields[idx].Notes = strings.TrimSpace(*patch.Notes) | ||||
| } | } | ||||
| if patch.SemanticSlot != nil { | |||||
| newSlot := strings.TrimSpace(*patch.SemanticSlot) | |||||
| if newSlot != fields[idx].SemanticSlot { | |||||
| fields[idx].SemanticSlot = newSlot | |||||
| if newSlot == "" { | |||||
| fields[idx].MappingSource = "" | |||||
| fields[idx].MappingConfidence = 0 | |||||
| } else { | |||||
| fields[idx].MappingSource = domain.MappingSourceManual | |||||
| fields[idx].MappingConfidence = 1.0 | |||||
| } | |||||
| } | |||||
| } | |||||
| } | } | ||||
| if err := s.manifestStore.UpdateFields(ctx, manifest.ID, fields); err != nil { | if err := s.manifestStore.UpdateFields(ctx, manifest.ID, fields); err != nil { | ||||
| @@ -262,6 +285,47 @@ func isLikelyImagePlaceholder(sample string) bool { | |||||
| return strings.Contains(normalized, " image image ") | return strings.Contains(normalized, " image image ") | ||||
| } | } | ||||
| // applyAISlotMapping runs the AI slot mapper (if configured) and writes the | |||||
| // resulting semantic slot + source onto the fields in-place. Failures are | |||||
| // logged and ignored — onboarding still succeeds, and rule-based mapping will | |||||
| // take over at build time. | |||||
| func (s *Service) applyAISlotMapping(ctx context.Context, templateID int64, fields []domain.TemplateField) { | |||||
| if s == nil || s.aiSlotMapper == nil || len(fields) == 0 { | |||||
| return | |||||
| } | |||||
| assignments, err := s.aiSlotMapper.MapFields(ctx, templateID, fields) | |||||
| if err != nil { | |||||
| slog.WarnContext(ctx, "ai slot mapping skipped", | |||||
| "component", "onboarding", | |||||
| "template_id", templateID, | |||||
| "error", err.Error(), | |||||
| ) | |||||
| return | |||||
| } | |||||
| if len(assignments) == 0 { | |||||
| return | |||||
| } | |||||
| bySlot := make(map[string]mapping.AISlotAssignment, len(assignments)) | |||||
| for _, a := range assignments { | |||||
| bySlot[a.FieldPath] = a | |||||
| } | |||||
| for i := range fields { | |||||
| assignment, ok := bySlot[strings.TrimSpace(fields[i].Path)] | |||||
| if !ok { | |||||
| continue | |||||
| } | |||||
| fields[i].SemanticSlot = assignment.Slot | |||||
| fields[i].MappingSource = domain.MappingSourceAI | |||||
| fields[i].MappingConfidence = assignment.Confidence | |||||
| } | |||||
| slog.InfoContext(ctx, "ai slot mapping applied", | |||||
| "component", "onboarding", | |||||
| "template_id", templateID, | |||||
| "mapped_count", len(assignments), | |||||
| "total_fields", len(fields), | |||||
| ) | |||||
| } | |||||
| func isLikelyImagePath(path string) bool { | func isLikelyImagePath(path string) bool { | ||||
| normalized := strings.ToLower(strings.TrimSpace(path)) | normalized := strings.ToLower(strings.TrimSpace(path)) | ||||
| for _, hint := range imageLikePathHints { | for _, hint := range imageLikePathHints { | ||||
| @@ -0,0 +1,8 @@ | |||||
| ALTER TABLE qc_template_fields | |||||
| ADD COLUMN semantic_slot TEXT NOT NULL DEFAULT ''; | |||||
| ALTER TABLE qc_template_fields | |||||
| ADD COLUMN mapping_source TEXT NOT NULL DEFAULT ''; | |||||
| ALTER TABLE qc_template_fields | |||||
| ADD COLUMN mapping_confidence REAL NOT NULL DEFAULT 0; | |||||
| @@ -189,10 +189,12 @@ func (s *Store) CreateManifest(ctx context.Context, manifest domain.TemplateMani | |||||
| _, err := tx.ExecContext(ctx, ` | _, err := tx.ExecContext(ctx, ` | ||||
| INSERT INTO qc_template_fields ( | INSERT INTO qc_template_fields ( | ||||
| id, template_id, manifest_id, section, website_section, key_name, path, field_kind, | id, template_id, manifest_id, section, website_section, key_name, path, field_kind, | ||||
| sample_value, is_enabled, is_required_by_us, display_label, display_order, notes | |||||
| ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, | |||||
| sample_value, is_enabled, is_required_by_us, display_label, display_order, notes, | |||||
| semantic_slot, mapping_source, mapping_confidence | |||||
| ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, | |||||
| f.ID, f.TemplateID, f.ManifestID, f.Section, domain.NormalizeWebsiteSection(f.WebsiteSection), f.KeyName, f.Path, f.FieldKind, | f.ID, f.TemplateID, f.ManifestID, f.Section, domain.NormalizeWebsiteSection(f.WebsiteSection), f.KeyName, f.Path, f.FieldKind, | ||||
| f.SampleValue, boolToInt(f.IsEnabled), boolToInt(f.IsRequiredByUs), f.DisplayLabel, f.DisplayOrder, f.Notes, | f.SampleValue, boolToInt(f.IsEnabled), boolToInt(f.IsRequiredByUs), f.DisplayLabel, f.DisplayOrder, f.Notes, | ||||
| strings.TrimSpace(f.SemanticSlot), strings.TrimSpace(f.MappingSource), f.MappingConfidence, | |||||
| ) | ) | ||||
| if err != nil { | if err != nil { | ||||
| return err | return err | ||||
| @@ -219,7 +221,8 @@ func (s *Store) GetActiveManifestByTemplateID(ctx context.Context, templateID in | |||||
| func (s *Store) ListFieldsByManifestID(ctx context.Context, manifestID string) ([]domain.TemplateField, error) { | func (s *Store) ListFieldsByManifestID(ctx context.Context, manifestID string) ([]domain.TemplateField, error) { | ||||
| rows, err := s.db.QueryContext(ctx, ` | rows, err := s.db.QueryContext(ctx, ` | ||||
| SELECT id, template_id, manifest_id, section, website_section, key_name, path, field_kind, sample_value, | SELECT id, template_id, manifest_id, section, website_section, key_name, path, field_kind, sample_value, | ||||
| is_enabled, is_required_by_us, display_label, display_order, notes | |||||
| is_enabled, is_required_by_us, display_label, display_order, notes, | |||||
| semantic_slot, mapping_source, mapping_confidence | |||||
| FROM qc_template_fields | FROM qc_template_fields | ||||
| WHERE manifest_id = ? | WHERE manifest_id = ? | ||||
| ORDER BY display_order ASC, id ASC`, manifestID) | ORDER BY display_order ASC, id ASC`, manifestID) | ||||
| @@ -235,12 +238,15 @@ func (s *Store) ListFieldsByManifestID(ctx context.Context, manifestID string) ( | |||||
| if err := rows.Scan( | if err := rows.Scan( | ||||
| &f.ID, &f.TemplateID, &f.ManifestID, &f.Section, &f.WebsiteSection, &f.KeyName, &f.Path, &f.FieldKind, &f.SampleValue, | &f.ID, &f.TemplateID, &f.ManifestID, &f.Section, &f.WebsiteSection, &f.KeyName, &f.Path, &f.FieldKind, &f.SampleValue, | ||||
| &isEnabled, &isRequired, &f.DisplayLabel, &f.DisplayOrder, &f.Notes, | &isEnabled, &isRequired, &f.DisplayLabel, &f.DisplayOrder, &f.Notes, | ||||
| &f.SemanticSlot, &f.MappingSource, &f.MappingConfidence, | |||||
| ); err != nil { | ); err != nil { | ||||
| return nil, err | return nil, err | ||||
| } | } | ||||
| f.IsEnabled = isEnabled == 1 | f.IsEnabled = isEnabled == 1 | ||||
| f.IsRequiredByUs = isRequired == 1 | f.IsRequiredByUs = isRequired == 1 | ||||
| f.WebsiteSection = domain.NormalizeWebsiteSection(f.WebsiteSection) | f.WebsiteSection = domain.NormalizeWebsiteSection(f.WebsiteSection) | ||||
| f.SemanticSlot = strings.TrimSpace(f.SemanticSlot) | |||||
| f.MappingSource = strings.TrimSpace(f.MappingSource) | |||||
| fields = append(fields, f) | fields = append(fields, f) | ||||
| } | } | ||||
| if err := rows.Err(); err != nil { | if err := rows.Err(); err != nil { | ||||
| @@ -273,10 +279,12 @@ func (s *Store) UpdateFields(ctx context.Context, manifestID string, fields []do | |||||
| _, err := tx.ExecContext(ctx, ` | _, err := tx.ExecContext(ctx, ` | ||||
| INSERT INTO qc_template_fields ( | INSERT INTO qc_template_fields ( | ||||
| id, template_id, manifest_id, section, website_section, key_name, path, field_kind, | id, template_id, manifest_id, section, website_section, key_name, path, field_kind, | ||||
| sample_value, is_enabled, is_required_by_us, display_label, display_order, notes | |||||
| ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, | |||||
| sample_value, is_enabled, is_required_by_us, display_label, display_order, notes, | |||||
| semantic_slot, mapping_source, mapping_confidence | |||||
| ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, | |||||
| f.ID, f.TemplateID, f.ManifestID, f.Section, domain.NormalizeWebsiteSection(f.WebsiteSection), f.KeyName, f.Path, f.FieldKind, | f.ID, f.TemplateID, f.ManifestID, f.Section, domain.NormalizeWebsiteSection(f.WebsiteSection), f.KeyName, f.Path, f.FieldKind, | ||||
| f.SampleValue, boolToInt(f.IsEnabled), boolToInt(f.IsRequiredByUs), f.DisplayLabel, f.DisplayOrder, f.Notes, | f.SampleValue, boolToInt(f.IsEnabled), boolToInt(f.IsRequiredByUs), f.DisplayLabel, f.DisplayOrder, f.Notes, | ||||
| strings.TrimSpace(f.SemanticSlot), strings.TrimSpace(f.MappingSource), f.MappingConfidence, | |||||
| ) | ) | ||||
| if err != nil { | if err != nil { | ||||
| return err | return err | ||||
| @@ -51,6 +51,7 @@ | |||||
| <th>Label</th> | <th>Label</th> | ||||
| <th>Order</th> | <th>Order</th> | ||||
| <th>Website Section</th> | <th>Website Section</th> | ||||
| <th>Semantic Slot</th> | |||||
| <th>Notes</th> | <th>Notes</th> | ||||
| <th>Sample</th> | <th>Sample</th> | ||||
| </tr> | </tr> | ||||
| @@ -74,6 +75,17 @@ | |||||
| {{end}} | {{end}} | ||||
| </select> | </select> | ||||
| </td> | </td> | ||||
| <td> | |||||
| <select name="field_semantic_slot_{{$i}}"> | |||||
| {{if and (ne $f.SemanticSlot "") (not $f.SemanticSlotIsKnown)}} | |||||
| <option value="{{$f.SemanticSlot}}" selected>{{$f.SemanticSlot}} (custom)</option> | |||||
| {{end}} | |||||
| {{range $opt := $.SemanticSlotOptions}} | |||||
| <option value="{{$opt.Value}}" {{if eq $f.SemanticSlot $opt.Value}}selected{{end}}>{{$opt.Label}}</option> | |||||
| {{end}} | |||||
| </select> | |||||
| {{if $f.MappingSource}}<small class="mono">{{$f.MappingSource}}</small>{{end}} | |||||
| </td> | |||||
| <td><input type="text" name="field_notes_{{$i}}" value="{{$f.Notes}}"></td> | <td><input type="text" name="field_notes_{{$i}}" value="{{$f.Notes}}"></td> | ||||
| <td class="mono">{{$f.SampleValue}}</td> | <td class="mono">{{$f.SampleValue}}</td> | ||||
| </tr> | </tr> | ||||