Skip to content

Commit bbc4eef

Browse files
VaughanJVaughanJ
committed
feat(jira): Allow extra JQL when querying JIRA board
1 parent 15e34bb commit bbc4eef

9 files changed

Lines changed: 290 additions & 19 deletions

File tree

‎backend/plugins/jira/impl/impl.go‎

Lines changed: 19 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,8 @@ import (
2525
"github.com/apache/incubator-devlake/core/dal"
2626
"github.com/apache/incubator-devlake/core/errors"
2727
coreModels "github.com/apache/incubator-devlake/core/models"
28+
"github.com/apache/incubator-devlake/core/models/domainlayer/crossdomain"
29+
"github.com/apache/incubator-devlake/core/models/domainlayer/didgen"
2830
"github.com/apache/incubator-devlake/core/plugin"
2931
helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
3032
"github.com/apache/incubator-devlake/plugins/jira/api"
@@ -195,8 +197,8 @@ func (p Jira) PrepareTaskData(taskCtx plugin.TaskContext, options map[string]int
195197
return nil, errors.Default.Wrap(err, "failed to create jira api client")
196198
}
197199

200+
var scope *models.JiraBoard
198201
if op.BoardId != 0 {
199-
var scope *models.JiraBoard
200202
// support v100 & advance mode
201203
// If we still cannot find the record in db, we have to request from remote server and save it to db
202204
db := taskCtx.GetDal()
@@ -249,6 +251,22 @@ func (p Jira) PrepareTaskData(taskCtx plugin.TaskContext, options map[string]int
249251
Options: &op,
250252
ApiClient: jiraApiClient,
251253
JiraServerInfo: *info,
254+
Board: scope,
255+
}
256+
257+
// Look up the DevLake project this board belongs to via project_mapping.
258+
// This is best-effort: if the board is not yet mapped (e.g. first run or
259+
// manual trigger outside a blueprint) we leave DevLakeProjectName empty.
260+
if scope != nil {
261+
domainBoardId := didgen.NewDomainIdGenerator(&models.JiraBoard{}).Generate(scope.ConnectionId, scope.BoardId)
262+
var pm crossdomain.ProjectMapping
263+
if lookupErr := db.First(&pm, dal.Where(
264+
"`table` = ? AND row_id = ?", "boards", domainBoardId,
265+
)); lookupErr == nil {
266+
taskData.DevLakeProjectName = pm.ProjectName
267+
} else if !db.IsErrorNotFound(lookupErr) {
268+
logger.Warn(lookupErr, "failed to look up project mapping for board %d", scope.BoardId)
269+
}
252270
}
253271

254272
return taskData, nil

‎backend/plugins/jira/models/migrationscripts/20260702_add_extra_jql_to_scope_config.go‎

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
1+
/*
2+
Licensed to the Apache Software Foundation (ASF) under one or more
3+
contributor license agreements. See the NOTICE file distributed with
4+
this work for additional information regarding copyright ownership.
5+
The ASF licenses this file to You under the Apache License, Version 2.0
6+
(the "License"); you may not use this file except in compliance with
7+
the License. You may obtain a copy of the License at
8+
9+
http://www.apache.org/licenses/LICENSE-2.0
10+
11+
Unless required by applicable law or agreed to in writing, software
12+
distributed under the License is distributed on an "AS IS" BASIS,
13+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
See the License for the specific language governing permissions and
15+
limitations under the License.
16+
*/
17+
18+
package migrationscripts
19+
20+
import (
21+
"github.com/apache/incubator-devlake/core/context"
22+
"github.com/apache/incubator-devlake/core/errors"
23+
"github.com/apache/incubator-devlake/helpers/migrationhelper"
24+
)
25+
26+
type JiraScopeConfig20260702 struct {
27+
ExtraJQL string `gorm:"type:varchar(255)"`
28+
}
29+
30+
func (JiraScopeConfig20260702) TableName() string {
31+
return "_tool_jira_scope_configs"
32+
}
33+
34+
type addExtraJQLToScopeConfig struct{}
35+
36+
func (script *addExtraJQLToScopeConfig) Up(basicRes context.BasicRes) errors.Error {
37+
return migrationhelper.AutoMigrateTables(basicRes, &JiraScopeConfig20260702{})
38+
}
39+
40+
func (*addExtraJQLToScopeConfig) Version() uint64 {
41+
return 20260702000000
42+
}
43+
44+
func (*addExtraJQLToScopeConfig) Name() string {
45+
return "add extra_jql to _tool_jira_scope_configs"
46+
}

‎backend/plugins/jira/models/migrationscripts/register.go‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -56,5 +56,6 @@ func All() []plugin.MigrationScript {
5656
new(updateScopeConfig),
5757
new(addFixVersions20250619),
5858
new(addSubQueryToBoards),
59+
new(addExtraJQLToScopeConfig),
5960
}
6061
}

‎backend/plugins/jira/models/scope_config.go‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@ package models
1919

2020
import (
2121
"regexp"
22+
"text/template"
2223

2324
"github.com/apache/incubator-devlake/core/errors"
2425
"github.com/apache/incubator-devlake/core/models/common"
@@ -49,6 +50,7 @@ type JiraScopeConfig struct {
4950
TypeMappings map[string]TypeMapping `mapstructure:"typeMappings,omitempty" json:"typeMappings" gorm:"type:json;serializer:json"`
5051
ApplicationType string `mapstructure:"applicationType,omitempty" json:"applicationType" gorm:"type:varchar(255)"`
5152
DueDateField string `mapstructure:"dueDateField,omitempty" json:"dueDateField" gorm:"type:varchar(255)"`
53+
ExtraJQL string `mapstructure:"extraJql,omitempty" json:"extraJql" gorm:"type:varchar(255)"`
5254
}
5355

5456
func (r *JiraScopeConfig) SetConnectionId(c *JiraScopeConfig, connectionId uint64) {
@@ -73,6 +75,11 @@ func (r *JiraScopeConfig) Validate() errors.Error {
7375
return errors.Convert(err)
7476
}
7577
}
78+
if r.ExtraJQL != "" {
79+
if _, tmplErr := template.New("extraJql").Funcs(template.FuncMap{}).Option("missingkey=error").Parse(r.ExtraJQL); tmplErr != nil {
80+
return errors.BadInput.Wrap(errors.Convert(tmplErr), "invalid ExtraJQL template")
81+
}
82+
}
7683
return nil
7784
}
7885

‎backend/plugins/jira/tasks/issue_collector.go‎

Lines changed: 76 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -18,12 +18,14 @@ limitations under the License.
1818
package tasks
1919

2020
import (
21+
"bytes"
2122
"encoding/json"
2223
"fmt"
2324
"io"
2425
"net/http"
2526
"net/url"
2627
"strings"
28+
"text/template"
2729
"time"
2830

2931
"github.com/apache/incubator-devlake/core/dal"
@@ -77,7 +79,15 @@ func CollectIssues(taskCtx plugin.SubTaskContext) errors.Error {
7779
// The board Agile API applies kanban sub-filters server-side, which silently
7880
// excludes resolved issues (e.g. those with a released fixVersion).
7981
// The search API with the saved filter JQL returns all matching issues.
80-
filterJql := buildFilterJQL(data.FilterId, incrementalJql)
82+
var extraJql string
83+
if data.Options.ScopeConfig != nil && data.Options.ScopeConfig.ExtraJQL != "" {
84+
renderedJql, renderErr := renderExtraJQL(data.Options.ScopeConfig.ExtraJQL, data)
85+
if renderErr != nil {
86+
return renderErr
87+
}
88+
extraJql = renderedJql
89+
}
90+
filterJql := buildFilterJQL(data.FilterId, extraJql, incrementalJql)
8191
logger.Info("collecting issues via search API with JQL: %s", filterJql)
8292

8393
pageSize := data.Options.PageSize
@@ -99,19 +109,73 @@ func CollectIssues(taskCtx plugin.SubTaskContext) errors.Error {
99109
return apiCollector.Execute()
100110
}
101111

102-
func buildFilterJQL(filterId string, incrementalJql string) string {
103-
if filterId == "" {
104-
return incrementalJql
112+
// JqlTemplateData holds the variables available inside an ExtraJQL template.
113+
// Users reference these with Go template syntax, e.g. `{{.BoardName}}`.
114+
type JqlTemplateData struct {
115+
BoardId uint64 // numeric ID of the connected Jira board
116+
BoardName string // display name of the connected Jira board
117+
DevLakeProjectName string // name of the DevLake project this board belongs to
118+
}
119+
120+
// renderExtraJQL executes the ExtraJQL scope-config field as a Go text/template,
121+
// substituting board-level variables so the same scope config can produce
122+
// different JQL for different boards.
123+
//
124+
// The template is parsed with an empty FuncMap (no built-in helpers such as
125+
// printf) and missingkey=error so that typos in variable names produce an
126+
// explicit error rather than silently rendering "<no value>".
127+
func renderExtraJQL(tmplStr string, data *JiraTaskData) (string, errors.Error) {
128+
tmpl, err := template.New("extraJql").
129+
Funcs(template.FuncMap{}).
130+
Option("missingkey=error").
131+
Parse(tmplStr)
132+
if err != nil {
133+
return "", errors.BadInput.Wrap(err, "invalid ExtraJQL template")
105134
}
106-
// Use Jira's `filter = {id}` syntax to reference the saved filter.
107-
// This avoids parenthesization bugs when composing raw JQL strings
108-
// that may contain OR/AND operators.
109-
if incrementalJql == "ORDER BY created ASC" {
110-
return fmt.Sprintf("filter = %s ORDER BY created ASC", filterId)
135+
136+
vars := JqlTemplateData{
137+
BoardId: data.Options.BoardId,
138+
DevLakeProjectName: data.DevLakeProjectName,
139+
}
140+
if data.Board != nil {
141+
vars.BoardName = data.Board.Name
142+
}
143+
144+
var buf bytes.Buffer
145+
if execErr := tmpl.Execute(&buf, vars); execErr != nil {
146+
return "", errors.BadInput.Wrap(execErr, "failed to render ExtraJQL template")
147+
}
148+
return buf.String(), nil
149+
}
150+
151+
// buildFilterJQL composes a final JQL query from three inputs:
152+
// - filterId: a Jira saved-filter ID (referenced via `filter = {id}`)
153+
// - extraJql: optional user-supplied JQL fragment appended as an AND condition
154+
// (e.g. `project = "MyComponent"`) to scope a large board down to one project
155+
// - incrementalJql: the time-based clause generated by buildJQL, always ending
156+
// with "ORDER BY created ASC"
157+
//
158+
// extraJql is wrapped in parentheses so that any OR/NOT operators inside it
159+
// do not interfere with the surrounding AND chain.
160+
func buildFilterJQL(filterId string, extraJql string, incrementalJql string) string {
161+
const orderBy = "ORDER BY created ASC"
162+
163+
var conditions []string
164+
if filterId != "" {
165+
conditions = append(conditions, fmt.Sprintf("filter = %s", filterId))
166+
}
167+
if extraJql != "" {
168+
conditions = append(conditions, fmt.Sprintf("(%s)", extraJql))
169+
}
170+
if incrementalJql != orderBy {
171+
// strip the trailing " ORDER BY created ASC" to isolate the time condition
172+
conditions = append(conditions, strings.TrimSuffix(incrementalJql, " "+orderBy))
173+
}
174+
175+
if len(conditions) == 0 {
176+
return orderBy
111177
}
112-
// incrementalJql contains "updated >= '...' ORDER BY created ASC"
113-
// We need to insert the filter reference before the incremental clause
114-
return fmt.Sprintf("filter = %s AND %s", filterId, incrementalJql)
178+
return strings.Join(conditions, " AND ") + " " + orderBy
115179
}
116180

117181
func setupIssueV2Collector(apiCollector *api.StatefulApiCollector, data *JiraTaskData, filterJql string, pageSize int) errors.Error {

‎backend/plugins/jira/tasks/issue_collector_test.go‎

Lines changed: 113 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,8 @@ package tasks
2020
import (
2121
"testing"
2222
"time"
23+
24+
"github.com/apache/incubator-devlake/plugins/jira/models"
2325
)
2426

2527
func Test_buildJQL(t *testing.T) {
@@ -66,6 +68,7 @@ func Test_buildFilterJQL(t *testing.T) {
6668
tests := []struct {
6769
name string
6870
filterId string
71+
extraJql string
6972
incrementalJql string
7073
want string
7174
}{
@@ -93,13 +96,122 @@ func Test_buildFilterJQL(t *testing.T) {
9396
incrementalJql: "updated >= '2024/01/01 00:00' ORDER BY created ASC",
9497
want: "updated >= '2024/01/01 00:00' ORDER BY created ASC",
9598
},
99+
{
100+
name: "extra jql with filter full sync",
101+
filterId: "12345",
102+
extraJql: `project = "MyComponent"`,
103+
incrementalJql: "ORDER BY created ASC",
104+
want: `filter = 12345 AND (project = "MyComponent") ORDER BY created ASC`,
105+
},
106+
{
107+
name: "extra jql with filter incremental sync",
108+
filterId: "12345",
109+
extraJql: `project = "MyComponent"`,
110+
incrementalJql: "updated >= '2024/01/01 00:00' ORDER BY created ASC",
111+
want: `filter = 12345 AND (project = "MyComponent") AND updated >= '2024/01/01 00:00' ORDER BY created ASC`,
112+
},
113+
{
114+
name: "extra jql without filter",
115+
filterId: "",
116+
extraJql: `project = "MyComponent"`,
117+
incrementalJql: "ORDER BY created ASC",
118+
want: `(project = "MyComponent") ORDER BY created ASC`,
119+
},
120+
{
121+
name: "extra jql without filter, incremental sync",
122+
filterId: "",
123+
extraJql: `project = "MyComponent"`,
124+
incrementalJql: "updated >= '2024/01/01 00:00' ORDER BY created ASC",
125+
want: `(project = "MyComponent") AND updated >= '2024/01/01 00:00' ORDER BY created ASC`,
126+
},
127+
{
128+
name: "extra jql with OR operator is parenthesized",
129+
filterId: "12345",
130+
extraJql: `project = "A" OR project = "B"`,
131+
incrementalJql: "ORDER BY created ASC",
132+
want: `filter = 12345 AND (project = "A" OR project = "B") ORDER BY created ASC`,
133+
},
96134
}
97135

98136
for _, tt := range tests {
99137
t.Run(tt.name, func(t *testing.T) {
100-
if got := buildFilterJQL(tt.filterId, tt.incrementalJql); got != tt.want {
138+
if got := buildFilterJQL(tt.filterId, tt.extraJql, tt.incrementalJql); got != tt.want {
101139
t.Errorf("buildFilterJQL() = %v, want %v", got, tt.want)
102140
}
103141
})
104142
}
105143
}
144+
145+
func Test_renderExtraJQL(t *testing.T) {
146+
makeData := func(boardId uint64, boardName string, projectName string) *JiraTaskData {
147+
return &JiraTaskData{
148+
Options: &JiraOptions{BoardId: boardId},
149+
Board: &models.JiraBoard{BoardId: boardId, Name: boardName},
150+
DevLakeProjectName: projectName,
151+
}
152+
}
153+
154+
tests := []struct {
155+
name string
156+
tmpl string
157+
data *JiraTaskData
158+
want string
159+
wantErr bool
160+
}{
161+
{
162+
name: "static JQL passes through unchanged",
163+
tmpl: `project = "MyProject"`,
164+
data: makeData(1, "My Board", ""),
165+
want: `project = "MyProject"`,
166+
},
167+
{
168+
name: "BoardName substitution",
169+
tmpl: `project = "{{.BoardName}}"`,
170+
data: makeData(42, "Team Alpha", ""),
171+
want: `project = "Team Alpha"`,
172+
},
173+
{
174+
name: "BoardId substitution",
175+
tmpl: `cf[10001] = {{.BoardId}}`,
176+
data: makeData(99, "Some Board", ""),
177+
want: `cf[10001] = 99`,
178+
},
179+
{
180+
name: "DevLakeProjectName substitution",
181+
tmpl: `component = "{{.DevLakeProjectName}}"`,
182+
data: makeData(1, "Big Shared Board", "payments-service"),
183+
want: `component = "payments-service"`,
184+
},
185+
{
186+
name: "nil Board falls back to empty BoardName",
187+
tmpl: `project = "{{.BoardName}}"`,
188+
data: &JiraTaskData{Options: &JiraOptions{BoardId: 1}, Board: nil},
189+
want: `project = ""`,
190+
},
191+
{
192+
name: "invalid template returns error",
193+
tmpl: `project = "{{.Unclosed"`,
194+
data: makeData(1, "My Board", ""),
195+
wantErr: true,
196+
},
197+
{
198+
name: "unknown field returns error (missingkey=error)",
199+
tmpl: `component = "{{.Typo}}"`,
200+
data: makeData(1, "My Board", ""),
201+
wantErr: true,
202+
},
203+
}
204+
205+
for _, tt := range tests {
206+
t.Run(tt.name, func(t *testing.T) {
207+
got, err := renderExtraJQL(tt.tmpl, tt.data)
208+
if (err != nil) != tt.wantErr {
209+
t.Errorf("renderExtraJQL() error = %v, wantErr %v", err, tt.wantErr)
210+
return
211+
}
212+
if !tt.wantErr && got != tt.want {
213+
t.Errorf("renderExtraJQL() = %v, want %v", got, tt.want)
214+
}
215+
})
216+
}
217+
}

‎backend/plugins/jira/tasks/task_data.go‎

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -34,10 +34,12 @@ type JiraOptions struct {
3434
}
3535

3636
type JiraTaskData struct {
37-
Options *JiraOptions
38-
ApiClient *api.ApiAsyncClient
39-
JiraServerInfo models.JiraServerInfo
40-
FilterId string
37+
Options *JiraOptions
38+
ApiClient *api.ApiAsyncClient
39+
JiraServerInfo models.JiraServerInfo
40+
FilterId string
41+
Board *models.JiraBoard
42+
DevLakeProjectName string
4143
}
4244

4345
type JiraApiParams models.JiraApiParams

0 commit comments

Comments
 (0)