Skip to content

Commit 12e9d57

Browse files
ksaricksaric
authored
[TSD-82] Fix failing tests. (#35)
1 parent b5f9163 commit 12e9d57

File tree

2 files changed

+59
-24
lines changed

2 files changed

+59
-24
lines changed

‎src/Lib.hs‎

Lines changed: 44 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@ module Lib
1616

1717
import Universum
1818

19+
import UnliftIO (MonadUnliftIO)
1920
import UnliftIO.Async (mapConcurrently)
2021
import UnliftIO.Concurrent (threadDelay)
2122

@@ -84,31 +85,53 @@ runZendeskMain = do
8485
ProcessTicketsFromTime fromTime -> runApp (processTicketsFromTime fromTime) cfg
8586
ShowStatistics -> void $ runApp (fetchTickets >>= showStatistics) cfg
8687
InspectLocalZip filePath -> runApp (inspectLocalZipAttachment filePath) cfg
87-
ExportData fromTime -> void $ runApp (exportZendeskDataToLocalDB fromTime) cfg
88+
ExportData fromTime -> void $ runApp (exportZendeskDataToLocalDB mapConcurrentlyWithDelay fromTime) cfg
89+
90+
-- | A general function for using concurrent calls.
91+
mapConcurrentlyWithDelay
92+
:: forall m a b. (MonadIO m, MonadUnliftIO m)
93+
=> [a] -> Int -> Int -> (a -> m b) -> m [b]
94+
mapConcurrentlyWithDelay dataIter chunksNum delayAmount concurrentFunction = do
95+
let chunkedData = chunks chunksNum dataIter
96+
97+
collectedData <- forM chunkedData $ \chunkedData' -> do
98+
-- Wait a minute!
99+
threadDelay delayAmount
100+
-- Concurrently we execute the function. If a single
101+
-- call fails, they all fail. When they all finish, they return the
102+
-- result.
103+
mapConcurrently concurrentFunction chunkedData'
104+
105+
pure . concat $ collectedData
106+
107+
-- | Yes, horrible. Seems like we need another layer, but I'm not convinced yet what it should be, so we
108+
-- wait patiently until it forms.
109+
type MapConcurrentlyFunction
110+
= [TicketInfo]
111+
-> Int
112+
-> Int
113+
-> (TicketInfo -> App (TicketInfo, [Comment]))
114+
-> App [(TicketInfo, [Comment])]
88115

89116
-- | The function for exporting Zendesk data to our local DB so
90117
-- we can have faster analysis and runs.
91118
-- We expect that the local database exists and has the correct schema.
92-
exportZendeskDataToLocalDB :: ExportFromTime -> App [TicketInfo]
93-
exportZendeskDataToLocalDB exportFromTime = do
119+
exportZendeskDataToLocalDB
120+
:: MapConcurrentlyFunction
121+
-> ExportFromTime
122+
-> App [TicketInfo]
123+
exportZendeskDataToLocalDB mapConcurrentlyWithDelay' exportFromTime = do
94124

95125
deleteAllData <- asksDBLayer dlDeleteAllData
96126

97127
notDeletedExportedTickets <- fetchTicketsExportedFromTime exportFromTime
98128

99-
-- We want to call in parallel 400 HTTP requests to fetch the data.
100-
let chunkedExportedTickets = chunks 400 notDeletedExportedTickets
101-
102-
-- The max requests are 400 per minute, so we wait a minute!
103-
ticketData <- forM chunkedExportedTickets $ \chunkedTickets -> do
104-
-- Wait a minute!
105-
threadDelay $ 61 * 1000000
106-
-- Concurrently we fetch the ticket data. If a single
107-
-- call fails, they all fail. When they all finish, they return the
108-
-- result.
109-
mapConcurrently fetchTicketData chunkedTickets
110-
111-
let allTicketData = concat ticketData
129+
-- Map concurrently if required.
130+
allTicketData <- mapConcurrentlyWithDelay'
131+
notDeletedExportedTickets
132+
400
133+
(60 * 1000000)
134+
fetchTicketData
112135

113136
-- Clear the data.
114137
deleteAllData
@@ -546,7 +569,11 @@ filterAnalyzedTickets ticketsInfo =
546569
&& isTicketInGoguenTestnet ticketInfo
547570

548571
analyzedTags :: [Text]
549-
analyzedTags = map renderTicketStatus [AnalyzedByScriptV1_0, AnalyzedByScriptV1_1, AnalyzedByScriptV1_2]
572+
analyzedTags = map renderTicketStatus
573+
[ AnalyzedByScriptV1_0
574+
, AnalyzedByScriptV1_1
575+
, AnalyzedByScriptV1_2
576+
]
550577

551578
isTicketAnalyzed :: TicketInfo -> Bool
552579
isTicketAnalyzed TicketInfo{..} = all (\analyzedTag -> analyzedTag `notElem` (getTicketTags tiTags)) analyzedTags

‎test/Spec.hs‎

Lines changed: 15 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -464,15 +464,15 @@ exportZendeskDataToLocalDBSpec =
464464
stubbedConfig = withStubbedIOAndZendeskLayer stubbedZendeskLayer
465465

466466
let appExecution :: IO [TicketInfo]
467-
appExecution = runApp (exportZendeskDataToLocalDB exportFromTime) stubbedConfig
467+
appExecution = runApp (exportZendeskDataToLocalDB mapNotConcurrent exportFromTime) stubbedConfig
468468

469469
ticketsToExport <- run appExecution
470470

471471
-- Check we don't have any tickets.
472472
assert . null $ ticketsToExport
473473

474474
it "should export a list of tickets since there are new tickets" $
475-
forAll (listOf1 arbitrary) $ \(listTickets) ->
475+
forAll (listOf1 genTicketWithUnsolvedStatus) $ \(listTickets) ->
476476
forAll arbitrary $ \(exportFromTime) ->
477477

478478
monadicIO $ do
@@ -489,7 +489,7 @@ exportZendeskDataToLocalDBSpec =
489489
stubbedConfig = withStubbedIOAndZendeskLayer stubbedZendeskLayer
490490

491491
let appExecution :: IO [TicketInfo]
492-
appExecution = runApp (exportZendeskDataToLocalDB exportFromTime) stubbedConfig
492+
appExecution = runApp (exportZendeskDataToLocalDB mapNotConcurrent exportFromTime) stubbedConfig
493493

494494
ticketsToExport <- run appExecution
495495

@@ -515,15 +515,15 @@ exportZendeskDataToLocalDBSpec =
515515
stubbedConfig = withStubbedIOAndZendeskLayer stubbedZendeskLayer
516516

517517
let appExecution :: IO [TicketInfo]
518-
appExecution = runApp (exportZendeskDataToLocalDB exportFromTime) stubbedConfig
518+
appExecution = runApp (exportZendeskDataToLocalDB mapNotConcurrent exportFromTime) stubbedConfig
519519

520520
ticketsToExport <- run appExecution
521521

522522
-- Check we don't have any tickets.
523523
assert . null $ ticketsToExport
524524

525525
it "should not return duplicated tickets" $
526-
forAll (listOf1 arbitrary) $ \(listTickets) ->
526+
forAll (listOf1 genTicketWithUnsolvedStatus) $ \(listTickets) ->
527527
forAll arbitrary $ \(exportFromTime) ->
528528

529529
monadicIO $ do
@@ -543,11 +543,19 @@ exportZendeskDataToLocalDBSpec =
543543
stubbedConfig = withStubbedIOAndZendeskLayer stubbedZendeskLayer
544544

545545
let appExecution :: IO [TicketInfo]
546-
appExecution = runApp (exportZendeskDataToLocalDB exportFromTime) stubbedConfig
546+
appExecution = runApp (exportZendeskDataToLocalDB mapNotConcurrent exportFromTime) stubbedConfig
547547

548548
ticketsToExport <- run appExecution
549549

550550
-- Check that we have tickets.
551551
assert . not . null $ ticketsToExport
552552
assert $ length ticketsToExport == length listTickets
553-
553+
where
554+
-- [a] -> Int -> Int -> (a -> m b) -> m [b]
555+
mapNotConcurrent
556+
:: [TicketInfo]
557+
-> Int
558+
-> Int
559+
-> (TicketInfo -> App (TicketInfo, [Comment]))
560+
-> App [(TicketInfo, [Comment])]
561+
mapNotConcurrent tickets _ _ fetchTicketData = forM tickets fetchTicketData

0 commit comments

Comments
 (0)