1. analytics

2. aboutme
whoami...
DavySuvee
@DSUVEE
➡ bigdataarchitect@datablend-continuum
• providebigdataandnosqlconsultancy
• 5yearsofhands-onexpertise

3. fromdatatoinsights

4. dataanalyticsinmongodb
chemicalsimilarityuse-case
nativeapi
aggregationframework
map/reduce

5. chemicalsimilarity(1)
★ 31millioncompoundsavailable
➡ pubchem
➡ Question:
★ findcompoundssimilartoa
particularothercompound

6. chemicalsimilarity(2)
0[N]1[C
O]2[C
C
C]
0[N]1[C
O]2[C
C
C]3[C
C
C
C
C]
0[C]1[C
C
C]2[C
C
N
O]3[C
C
C
C
O
O]
0[C]1[C
C]2[C
C
C
C
O]3[C
C
N
O]
0[O]1[C]2[C
O]3[C
C
C]
0[C]1[C
O
O]2[C
C
C
O]
0[C]1[C
C]2[C
C]
0[C]1[C]2[C]3[C
O]
0[C]1[C
C
N]2[C
C
C
C
O]3[C
C
C
O]
...

7. chemicalsimilarity(3)
0[N]1[C
O]2[C
C
C]
0[N]1[C
O]2[C
C
C]3[C
C
C
C
C]
0[C]1[C
C
C]2[C
C
N
O]3[C
C
C
C
O
O]
0[C]1[C
C]2[C
C
C
C
O]3[C
C
N
O]
0[O]1[C]2[C
O]3[C
C
C]
0[C]1[C
O
O]2[C
C
C
O]
0[C]1[C
C]2[C
C]
0[C]1[C]2[C]3[C
O]
0[C]1[C
C
N]2[C
C
C
C
O]3[C
C
C
O]
...
0[N]1[C
O]2[C
C
C]3[C
C
C
C
C
C]
0[C]1[C
C
C]2[C
C
N
O]3[C
C
C
C
O
O]
0[C]1[C
C]2[C
C
C
C
O]3[C
C
N
O]
0[O]1[C]2[C
O]3[C
C
C
C]
0[C]1[C
O
O]2[C
C
C
O]
0[C]1[C
C]2[C
C]
0[N]1[C
O]2[C
C
C]
0[C]1[C]2[C]3[C
O]
0[C]1[C
C
N]2[C
C
C
C
O]3[C
C
C
O]
...
equalityviatanimoto
but31millioncalculations?

8. mongodbdatamodel(1)
{
"compound_cid"
:
"46200001"
,
"smiles"
:
"CCC1C(C(C(C(=NOCC=CCN2CCCCC2)C(CC(C(C(C(C(C(=O)O1)C)OC3C"
,
"fingerprint_count"
:
120
,
"fingerprints"
:
[
"0[N]1[C
O]2[C
C
C]"
,
"0[N]1[C
O]2[C
C
C]3[C
C
C
C
C]"
,
"0[C]1[C
C
C]2[C
C
N
O]3[C
C
C
C
O
O]"
,
"0[C]1[C
C]2[C
C
C
C
O]3[C
C
N
O]"
,
"0[O]1[C]2[C
O]3[C
C
C]"
,
"0[C]1[C
O
O]2[C
C
C
O]"
,
"0[C]1[C
C]2[C
C]"
,
"0[C]1[C]2[C]3[C
O]"
,
"0[C]1[C
C
N]2[C
C
C
C
O]3[C
C
C
O]"
,
...
]
,
}
compound
collection

9. mongodbdatamodel(2) fingerprint
collection
{
"fingerprint"
:
"0[N]1[C
O]2[C
C
C]",
"count"
:
472
}
{
"fingerprint"
:
"0[N]1[C
O]2[C
C
C]3[C
C
C
C
C]",
"count"
:
41
}
{
"fingerprint"
:
"0[O]1[C]2[C
O]3[C
C
C]",
"count"
:
1343
}

10. queryingpattern(1)
★ from31million->potentialmatch
➡ narrowdownthesearchspace
➡ imagine80%searchforacompoundwith40features
➡ 32
➡ 50

11. queryingpattern(2)
★ from31million->potentialmatch
➡ narrowdownthesearchspace
➡ imagine80%searchforacompoundwith40features
(9fingerprints)

12. findthefingerprints(1)
//
Retrieve
the
particular
compound
DBObject
object
=
compoundsCollection.findOne(QueryBuilder.start("compound_cid").is(compound).get());
//
Retrieve
the
relevant
properties
String
pubchemcid
=
(String)object.get(COMPOUNDCID_PROPERTY);
List<Integer>
fingerprintstofind
=
Arrays.asList(((BasicDBList)object.get(FINGERPRINTS_PROPERTY)).toArray(new
Integer[]{}));
//
Sort
the
fingerprints
on
total
number
of
occurences
fingerprintstofind
=
findSortedFingerprints(fingerprintstofind);

13. findthefingerprints(2)
List<Integer>
sortedFingerprintsToFind
=
new
ArrayList<Integer>();
//
Find
all
fingerprint
count
documents
DBObject
fingerprintcountquery
=
QueryBuilder.start(FINGERPRINT_PROPERTY).in(fingerprintsToFind.toArray()).get();
//
Only
retrieve
the
fingerprint
string
itself
DBObject
fingerprintcountselection
=
QueryBuilder.start(FINGERPRINT_PROPERTY).is(1).get();
//
Sort
the
result
on
count
DBObject
fingerprintcountsort
=
QueryBuilder.start(COUNT_PROPERTY).is(1).get();
//
Execute
the
query
on
the
fingerprint
counts
collection
DBCursor
fingerprintcounts
=
fingerprintCountsCollection.find(fingerprintcountquery,
fingerprintcountselection).
sort(fingerprintcountsort);

14. nativequery(1)
//
Find
the
matching
compounds
DBObject
compoundquery
=
QueryBuilder.
start(FINGERPRINTS_PROPERTY).
in(fingerprintsToConsider).
and(FINGERPRINTCOUNT_PROPERTY).lessThanEquals(maxnumberofcompoundfingerprints).
and(FINGERPRINTCOUNT_PROPERTY).greaterThanEquals(minnumberofcompoundfingerprints).
get();

15. nativequery(2)
//
Execute
the
query
DBCursor
compounds
=
compoundsCollection.find(compoundquery);
//
Let's
process
the
found
compounds
locally
while(compounds.hasNext())
{
DBObject
compound
=
compounds.next();
BasicDBList
fingerprints
=
((BasicDBList)
compound.get(FINGERPRINTS_PROPERTY));
//
Calculate
the
intersection
on
the
total
list
of
fingerprints
fingerprints.retainAll(fingerprintsToFind);
if
(fingerprints.size()
>=
minnumberofcompoundfingerprints)
{
//
Calculate
the
tanimoto
coefficient
...
}
}

16. map/reducequery(1)

17. map/reducequery(2)
//
Find
all
compounds
DBObject
compoundquery
=
...
//
The
map
fuction
String
map
=
"function()
{
"
+
"
var
found
=
0;
"
+
"
var
fingerprintslength
=
this.fingerprints.length;
"
+
"
for
(i
=
0;
i
<
fingerprintslength;
i++)
{
"
+
"
if
(fingerprintstofind[this.fingerprints[i]]
===
true)
{
found++;
}
"
+
"
}
"
+
"
if
(found
>=
minnumberofcompoundfingerprints)
{
"
+
"
emit
(this.compound_cid,
{found
:
found,
"
+
"
total
:
this.fingerprint_count}
);
}
"
+
"}";
//
Execute
the
map
reduce
function
MapReduceCommand
mr
=
new
MapReduceCommand(compoundsCollection,
map,
"",
MapReduceCommand.OutputType.INLINE,
compoundquery);

18. aggregationframework(1)

19. aggregationframework(2)
{
"aggregate"
:
"compounds"
,
"pipeline"
:
[
{
"$match"
:
{
"fingerprint_count"
:
{
"$gte"
:
4
,
"$lte"
:
1780}}}
,
{
"$unwind"
:
"$fingerprints"}
,
{
"$match"
:
{
"fingerprints"
:
{
"$in"
:
[
1960,
15111,
...,94
,
26]}}}
,
{
"$group"
:
{
"_id"
:
"$compound_cid"
,
"fingerprintmatches"
:
{
"$sum"
:
1}
,
"totalcount"
:
{
"$first"
:
"$fingerprint_count"}
}}}
,
{
"$project"
:
{
"_id"
:
1
,
"tanimoto"
:
{
"$divide"
:
[
"$fingerprintmatches"
,
{
"$subtract"
:
[
{
"$add"
:
[
89
,
"$totalcount"]}
,
"$fingerprintmatches"]}]}}
,
{
"$match"
:
{
"tanimoto"
:
{
"$gte"
:
0.05}}}]}

20. benchmarkresults
★ native->202ms
➡ 100Kcompounds,0.8tanimoto
★ map/reduce->214ms
★ aggregationframework->609ms
★ native->1909ms
➡ 100Kcompounds,0.05tanimoto
★ map/reduce->20978ms
★ aggregationframework->1613ms

21. diymongodbanalytics...
➡ http://datablend.be/?p=256
➡ thejoyofalgorithmsandnosql:amongodbexample
➡ http://github.com/datablend/mongo-compound-comparison-revisited

22. Questions?

23. E-MAIL
info@datablend.be
Followus
twitter.com/data_blend
www.datablend.be
www.datablend.be info@datablend.be 0499/05.00.89
datablend-continuum