Repository navigation
Expand file tree
/
Copy pathsearch.clj
More file actions
158 lines (137 loc) · 6.34 KB
/
Copy pathsearch.clj
File metadata and controls
158 lines (137 loc) · 6.34 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
(ns leiningen.search
(:require [clojure.java.io :as io]
[clojure.string :as string]
[leiningen.core.user :as user]
[leiningen.core.main :as main]
[leiningen.core.project :as project]
[clj-http.client :as http])
(:import (org.apache.maven.index IteratorSearchRequest MAVEN NexusIndexer)
(org.apache.maven.index.context IndexingContext)
(org.apache.maven.index.creator JarFileContentsIndexCreator
MavenPluginArtifactInfoIndexCreator
MinimalArtifactInfoIndexCreator)
(org.apache.maven.index.expr UserInputSearchExpression)
(org.apache.maven.index.updater IndexUpdater IndexUpdateRequest
ResourceFetcher)
(org.codehaus.plexus DefaultPlexusContainer PlexusContainer)))
(defonce container (DefaultPlexusContainer.))
(defonce indexer (.lookup container NexusIndexer))
(def ^:private default-indexers [(MinimalArtifactInfoIndexCreator.)
(JarFileContentsIndexCreator.)
(MavenPluginArtifactInfoIndexCreator.)])
(defn index-location [url]
(io/file (user/leiningen-home) "indices" (string/replace url #"[:/]" "_")))
(defn- add-context [[id {:keys [url]}]]
(.addIndexingContextForced indexer id url nil (index-location url)
url nil default-indexers))
(defn- remove-context [context]
(.removeIndexingContext indexer context false))
;; TODO: add progress reporting back in
(defn- http-resource-fetcher []
(let [base-url (promise)
stream (promise)]
(proxy [ResourceFetcher] []
;; TODO: handle connect/disconnect properly
(connect [id url]
(deliver base-url url))
(disconnect []
(.close @stream))
(^java.io.InputStream retrieve [name]
(main/debug "Downloading" (str @base-url "/" name))
(let [s (:body (http/get (str @base-url "/" name)
{:throw-exceptions false :as :stream}))]
(deliver stream s)
s)))))
(defn update-index [context]
(.fetchAndUpdateIndex (.lookup container IndexUpdater)
(IndexUpdateRequest. context (http-resource-fetcher))))
(defn- refresh? [url project]
(if-not (:offline? project)
(< (.lastModified (io/file (index-location url) "timestamp"))
(- (System/currentTimeMillis) 86400000))))
(defn- parse-result [result]
(let [group-id (.groupId result)
artifact-id (.artifactId result)
version (.version result)
classifier (.classifier result)
packaging (.packaging result)
name (if (= group-id artifact-id)
(symbol artifact-id)
(symbol group-id artifact-id))
classifier-opts (and classifier [:classifier classifier])
packaging-opts (if (not= "jar" packaging) [:packaging packaging])]
[(pr-str (into [name version] (concat classifier-opts packaging-opts)))
(or (.description result) "")]))
(def ^:private page-size (:search-page-size (:user (user/profiles)) 50))
(defn- print-results [response page]
(when (seq response)
(println " == Showing page" page "/"
(-> (.getTotalHitsCount response) (/ page-size) Math/ceil int))
(doseq [[dep description] (map parse-result response)]
(println dep description))
(println)))
(def ^{:private true}
field-splitter-re #"^([a-z]+)(?:\:)(.+)")
(defn- lookup-lucene-field-for
[^String s]
(case (-> (or s "") .toLowerCase .trim)
"" MAVEN/ARTIFACT_ID
"id" MAVEN/ARTIFACT_ID
"a" MAVEN/ARTIFACT_ID
"artifact-id" MAVEN/ARTIFACT_ID
"artifact_id" MAVEN/ARTIFACT_ID
"g" MAVEN/GROUP_ID
"group" MAVEN/GROUP_ID
"group-id" MAVEN/GROUP_ID
"group_id" MAVEN/GROUP_ID
"d" MAVEN/DESCRIPTION
"desc" MAVEN/DESCRIPTION
"description" MAVEN/DESCRIPTION
(throw (IllegalArgumentException. (format "search over the field %s is not supported; known fields: id, description (aliased as d), group (aliased as g)" s)))))
(defn- split-query
"Splits \"field:query\" into \"field\" and \"query\""
[^String s]
(let [[_ field query] (re-find field-splitter-re s)]
[(lookup-lucene-field-for field)
(or query s)]))
(defn search-repository [query contexts page]
(let [[field q] (split-query query)
search-expression (UserInputSearchExpression. q)
constructed-query (.constructQuery indexer field
search-expression)
request (doto (IteratorSearchRequest. constructed-query contexts)
(.setStart (* (dec page) page-size))
(.setCount page-size))]
(with-open [response (.searchIterator indexer request)]
(println (format "Searching over %s..." (.getDescription field)))
(print-results response page))))
(defn ^:no-project-needed search
"Search remote maven repositories for matching jars.
The first run will download a set of indices, which will take a very long time.
The query is evaluated as a lucene search. You can search for simple string
matches or do more advanced queries such as this:
$ lein search clojure
$ lein search description:crawl
$ lein search group:clojurewerkz
$ lein search \"Riak client\"
Also accepts a second parameter for fetching successive pages."
([project query] (search project query 1))
([project query page]
;; Maven's indexer requires over 1GB of free space for a <100M index
(let [orig-tmp (System/getProperty "java.io.tmpdir")
new-tmp (io/file (user/leiningen-home) "indices" "tmp")
project (or project (project/make {}))
contexts (doall (map add-context (:repositories project)))]
(try
(.mkdirs new-tmp)
(System/setProperty "java.io.tmpdir" (str new-tmp))
(doseq [context contexts]
(when (refresh? (.getRepositoryUrl context) project)
(do
(println "Updating the search index. This may take a few minutes...")
(update-index context))))
;; TODO: improve error message when page isn't numeric
(search-repository query contexts (Integer. page))
(finally
(doall (map remove-context contexts))
(System/setProperty "java.io.tmpdir" orig-tmp))))))