Prefetch next page in cache

backend/Cache.dcl

@@ -4,8 +4,10 @@ from StdOverloaded import class toString
 from Data.Maybe import :: Maybe
 from Text.JSON import generic JSONEncode, generic JSONDecode, :: JSONNode
 
+:: CacheType = Brief | LongTerm
+
 // Check if for the hash of the argument a JSON file exists of type b
-readCache :: !a -> *World -> (Maybe b, !*World) | toString a & JSONDecode{|*|} b
+readCache :: !a *World -> (Maybe b, !*World) | toString a & JSONDecode{|*|} b
 
 // Write for the hash of a a JSON file of type b
-writeCache :: !a !b -> *World -> *World | toString a & JSONEncode{|*|} b
+writeCache :: CacheType !a !b -> *World -> *World | toString a & JSONEncode{|*|} b

backend/Cache.icl

@@ -14,13 +14,19 @@ import System.FilePath
 import System.File
 import Data.Tuple
 
-CACHE_DIR :== "./cache"
+cache_types :== [Brief, LongTerm]
 
-toCacheFile :: (a -> FilePath) | toString a
-toCacheFile = ((</>) CACHE_DIR) o md5 o toString
+cache_dir :: CacheType -> FilePath
+cache_dir LongTerm = "./cache/lt"
+cache_dir Brief = "./cache/brief"
 
-readCache :: !a -> *World -> (Maybe b, !*World) | toString a & JSONDecode{|*|} b
-readCache k = appFst (join o fmap (fromJSON o fromString) o error2mb) o readFile (toCacheFile k)
+toCacheFile :: CacheType -> a -> FilePath | toString a
+toCacheFile t = (</>) (cache_dir t) o md5 o toString
 
-writeCache :: !a !b -> *World -> *World | toString a & JSONEncode{|*|} b
-writeCache k v = snd o writeFile (toCacheFile k) (toString $ toJSON v)
+readCache :: !a *World -> (Maybe b, !*World) | toString a & JSONDecode{|*|} b
+readCache k w
+# (files,w) = seqList [appFst error2mb o readFile (toCacheFile t k) \\ t <- cache_types] w
+= (join o fmap (fromJSON o fromString) $ foldl (<|>) empty files, w)
+
+writeCache :: CacheType !a !b -> *World -> *World | toString a & JSONEncode{|*|} b
+writeCache t k v = snd o writeFile (toCacheFile t k) (toString $ toJSON v)

backend/CloogleServer.icl

@@ -131,6 +131,7 @@ E_INVALIDNAME  :== 129
 E_INVALIDTYPE  :== 130
 
 MAX_RESULTS    :== 15
+CACHE_PREFETCH :== 5
 
 Start w
 # (io, w) = stdio w
@@ -173,7 +174,7 @@ where
 			= sortBy (\a b -> snd a > snd b) <$>
 			  filter ((<)(length results) o snd) <$>
 			  (mbType >>= \t -> suggs name t db)
-		# results = take MAX_RESULTS results
+		# (results,nextpages) = splitAt MAX_RESULTS results
 		// Response
 		# response = if (isEmpty results)
 			(err E_NORESULTS "No results")
@@ -183,11 +184,28 @@ where
 		    , more_available = Just more
 		    , suggestions    = suggestions
 		    }
+		// Save page prefetches
+		# w = cachePages CACHE_PREFETCH 1 response nextpages w
 		// Save cache file
 		= respond response w
 	where
 		respond :: Response *World -> *(Response, *World)
-		respond r w = (r, writeCache request r w)
+		respond r w = (r, writeCache LongTerm request r w)
+
+		cachePages :: Int Int Response [Result] *World -> *World
+		cachePages _ _  _ [] w = w
+		cachePages 0 _  _ _  w = w
+		cachePages npages i response results w
+		# w = writeCache Brief req` resp` w
+		= cachePages (npages - 1) (i + 1) response keep w
+		where
+			req` = { request & page = ((+) i) <$> request.page <|> pure 0 }
+			resp` =
+				{ response
+				& more_available = Just $ max 0 (length results - MAX_RESULTS)
+				, data = give
+				}
+			(give,keep) = splitAt MAX_RESULTS results
 
 	suggs :: !(Maybe String) !Type !TypeDB -> Maybe [(Request, Int)]
 	suggs n (Func is r cc) db

backend/Dockerfile

@@ -19,7 +19,6 @@ COPY . /usr/src/cloogle
 WORKDIR /usr/src/cloogle
 
 RUN make distclean && make
-RUN mkdir -p ./cache
 
 EXPOSE 31215
 

backend/serve

 #!/bin/sh
+mkdir -p ./cache/lt ./cache/brief
 while :
 do
 	< types.json \

gc/gc.sh

@@ -6,11 +6,20 @@ CACHE_DIR=/var/cache
 cd "$CACHE_DIR"
 
 while :; do
-	n="$(($(ls -l | wc -l)-1))"
+	# Long term cache
+	cd lt
+	n="$(($(ls -1 | wc -l)-1))"
 	if [ "$n" -gt "$CACHE_SIZE" ]; then
 		ls -1tu | tail -n "$((n-CACHE_SIZE))" | xargs -P$(nproc) -r rm -v
 	else
 		echo "$n / $CACHE_SIZE cache entries used"
 	fi
+	cd ..
+	# Brief cache
+	cd brief
+	echo "$(($(ls -1 | wc -l)-1)) entries removed from brief cache"
+	rm *
+	cd ..
+	# Wait
 	sleep $INTERVAL
 done