Merge branch 'test' into 'main'

add test suite for unique devices

See merge request repos/generated-data-platform/aqs/unique-devices!4

go.mod

@@ -9,7 +9,7 @@ require (
 	github.com/julienschmidt/httprouter v1.3.0
 	github.com/prometheus/client_golang v1.10.0
 	github.com/stretchr/testify v1.7.0
-	gitlab.wikimedia.org/frankie/aqsassist v0.0.0-20220826183849-400a9e2422b4
+	gitlab.wikimedia.org/frankie/aqsassist v0.0.0-20221004182421-b725841c67df
 	gopkg.in/yaml.v2 v2.4.0
 	schneider.vip/problem v1.6.0
 )

go.sum

@@ -278,6 +278,8 @@ github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtX
 github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
 gitlab.wikimedia.org/frankie/aqsassist v0.0.0-20220826183849-400a9e2422b4 h1:AlLmD7C8ZP8ZoqKzgfYSXqRGtxY2n+xvD4hqaPcgsKA=
 gitlab.wikimedia.org/frankie/aqsassist v0.0.0-20220826183849-400a9e2422b4/go.mod h1:N6+tSx/RYnd/sc3rq41E4Q9CvK8QG3lGrp0fn3BbqmI=
+gitlab.wikimedia.org/frankie/aqsassist v0.0.0-20221004182421-b725841c67df h1:3WpPiJmOdkYs1zflKUvMi2XFBJBwBsA0c1LJm8kOTyk=
+gitlab.wikimedia.org/frankie/aqsassist v0.0.0-20221004182421-b725841c67df/go.mod h1:N6+tSx/RYnd/sc3rq41E4Q9CvK8QG3lGrp0fn3BbqmI=
 go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
 go.etcd.io/etcd v0.0.0-20191023171146-3cf2f69b5738/go.mod h1:dnLIgRNXwCJa5e+c6mIZCrds/GIG4ncV9HhK5PX7jPg=
 go.opencensus.io v0.20.1/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk=

unique-devices.go → unique_devices.go

@@ -36,6 +36,7 @@ type UniqueDevicesHandler struct {
 }
 
 func (s *UniqueDevicesHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+
 	var err error
 	var params = httprouter.ParamsFromContext(r.Context())
 	var response = UniqueDevicesResponse{Items: make([]UniqueDevices, 0)}
@@ -66,6 +67,7 @@ func (s *UniqueDevicesHandler) ServeHTTP(w http.ResponseWriter, r *http.Request)
 			problem.Custom("uri", r.RequestURI)).WriteTo(w)
 		return
 	}
+
 	if end, err = aqsassist.ValidateTimestamp(params.ByName("end")); err != nil {
 		problem.New(
 			problem.Type("about:blank"),
@@ -77,6 +79,17 @@ func (s *UniqueDevicesHandler) ServeHTTP(w http.ResponseWriter, r *http.Request)
 		return
 	}
 
+	if err = aqsassist.StartBeforeEnd(start, end); err != nil {
+		problem.New(
+			problem.Type("about:blank"),
+			problem.Title(http.StatusText(http.StatusBadRequest)),
+			problem.Custom("method", http.MethodGet),
+			problem.Status(http.StatusBadRequest),
+			problem.Detail("Invalid timestamp"),
+			problem.Custom("uri", r.RequestURI)).WriteTo(w)
+		return
+	}
+
 	ctx := context.Background()
 
 	query := `SELECT devices, offset, underestimate, timestamp FROM "local_group_default_T_unique_devices".data WHERE "_domain" = 'analytics.wikimedia.org' AND project = ? AND "access-site" = ? AND granularity = ? AND timestamp >= ? AND timestamp <= ?`

unique_devices_test.go

+package main
+
+import (
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+
+	"net/http"
+	"os"
+	"strconv"
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+//unique devices accepts timestamps of YYYYMMDD and YYYYMMDDHH
+//does not require full month in range: 20210101 to 20210102 returns monthly data
+
+func testURL(suffix string) string {
+	var fallback = "http://localhost:8080/metrics/unique-devices"
+	var res string
+	if res = os.Getenv("API_URL"); res == "" {
+		return fmt.Sprintf("%s/%s", fallback, suffix)
+	}
+	return fmt.Sprintf("%s/%s", strings.TrimRight(res, "/"), suffix)
+}
+
+func runQuery(t *testing.T, project string, sites string, granularity string, start int, end int) UniqueDevicesResponse {
+
+	res, err := http.Get(testURL(fmt.Sprintf("%s/%s/%s/%s/%s", project, sites, granularity, strconv.FormatInt(int64(start), 10), strconv.FormatInt(int64(end), 10))))
+
+	require.NoError(t, err, "Invalid http request")
+
+	require.Equal(t, http.StatusOK, res.StatusCode, "Wrong status code")
+
+	body, err := ioutil.ReadAll(res.Body)
+	require.NoError(t, err, "Unable to read response")
+
+	n := UniqueDevicesResponse{}
+	err = json.Unmarshal(body, &n)
+
+	require.NoError(t, err, "Unable to unmarshal response body")
+	return n
+}
+
+func TestUniqueDevices(t *testing.T) {
+	t.Run("should return 200 for expected parameters", func(t *testing.T) {
+
+		res, err := http.Get(testURL("/en.wikipedia/all-sites/daily/20210101/20210201"))
+
+		require.NoError(t, err, "Invalid http request")
+
+		require.Equal(t, http.StatusOK, res.StatusCode, "Wrong status code")
+	})
+
+	t.Run("should return 400 when parameters are wrong", func(t *testing.T) {
+
+		res, err := http.Get(testURL("/wrong-project/wrong-sites/wrong-granularity/00000000/00000000"))
+
+		require.NoError(t, err, "Invalid http request")
+
+		require.Equal(t, http.StatusBadRequest, res.StatusCode, "Wrong status code")
+	})
+
+	t.Run("should return 400 when start is after end", func(t *testing.T) {
+
+		res, err := http.Get(testURL("/en.wikipedia/all-sites/daily/20210201/20210101"))
+
+		require.NoError(t, err, "Invalid http request")
+
+		require.Equal(t, http.StatusBadRequest, res.StatusCode, "Wrong status code")
+
+	})
+
+	t.Run("should return 400 when timestamp is invalid", func(t *testing.T) {
+
+		res, err := http.Get(testURL("/en.wikipedia/all-sites/daily/0000/0000"))
+
+		require.NoError(t, err, "Invalid http request")
+
+		require.Equal(t, http.StatusBadRequest, res.StatusCode, "Wrong status code")
+
+	})
+
+	t.Run("should return the same data when using timestamps with hours", func(t *testing.T) {
+
+		n := runQuery(t, "en.wikipedia", "all-sites", "daily", 2021010100, 2021020100)
+
+		assert.Len(t, n.Items, 31, "Unexpected response length")
+
+		u := UniqueDevices{
+			Project:       "en.wikipedia",
+			AccessSite:    "all-sites",
+			Granularity:   "daily",
+			Timestamp:     "20210102",
+			Devices:       75002648,
+			Offset:        14784457,
+			Underestimate: 60218191,
+		}
+
+		assert.Equal(t, n.Items[0], u, "Wrong contents")
+
+	})
+
+	t.Run("should include offset and underestimate", func(t *testing.T) {
+
+		n := runQuery(t, "en.wikipedia", "all-sites", "daily", 20210101, 20210201)
+
+		assert.Equal(t, n.Items[0].Offset, 14784457, "Wrong contents")
+		assert.Equal(t, n.Items[0].Underestimate, 60218191, "Wrong contents")
+
+	})
+
+	t.Run("should return numeric values as integers", func(t *testing.T) {
+
+		n := runQuery(t, "en.wikipedia", "all-sites", "daily", 20210101, 20210201)
+
+		assert.IsType(t, 0, n.Items[0].Devices, "Wrong type")
+		assert.IsType(t, 0, n.Items[0].Offset, "Wrong type")
+		assert.IsType(t, 0, n.Items[0].Underestimate, "Wrong type")
+
+	})
+
+}