NandanM Tracker

Re: Proc Groovy\HTTP Headaches

NandanM — Wed, 16 Jul 2014 03:41:03 GMT

I tried running the code below, to retrieve 1000 tweets by changing the count value to 1000 and changing the _n_ value of the do loop. In the output, I only see 47 tweets. I was wondering what “rc” indicates in this code.

When I comment /*tweets.callBooleanMethod("hasNext",rc);*/,and run the program, I get 1000 observations. However, after the 47th observation, all the remaining observations until the 1000th one is simply the 47th line repeated.

/*-----------------------------------------------------------------------------------------------------------------------------------------

*-Usage Parameters

%let api_key =

%let api_secret =

%let search_query = %23AMAZONFIRE+OR+%23AMAZONSMARTPHONE;

/*-----------------------------------------------------------------------------------------------------------------------------------------

*-Collect Ivy Jar

filename ivy "%sysfunc(pathname(work,l))/ivy.jar";

proc http

method = "get"

url = "http://central.maven.org/maven2/org/apache/ivy/ivy/2.3.0-rc1/ivy-2.3.0-rc1.jar"

out = ivy;

run;

/*-----------------------------------------------------------------------------------------------------------------------------------------

*-Compile GROOVY/JAVA Classes

filename cp temp;

proc groovy classpath=cp;

*add the ivy jar just downloaded and include in classpath;

*we need this to use @Grab notation below for additional dependencies;

add classpath=ivy;

*the following line requires modification based on OS and SAS Version, below accurate for 9.4M1 on Linux 64;

*this was necessary because JsonSlurper was not available initially;

*alternatively, you could collect this with @Grab, should you groovy-all.jar not contain this module;

* @Grab(group='org.codehaus.groovy', module='groovy-json', version='2.1.3');

add classpath="C:\Users\Resh\Downloads\groovy-all-2.3.4.jar";

submit parseonly;

@Grab(group='org.codehaus.groovy.modules.http-builder', module='http-builder', version='0.7')

@Grab(group='commons-codec', module='commons-codec', version='1.2')

import groovy.json.JsonSlurper

import org.apache.commons.codec.binary.Base64

import groovyx.net.http.HTTPBuilder

import static groovyx.net.http.ContentType.JSON

import static groovyx.net.http.Method.GET

import static groovyx.net.http.Method.POST

class TwitterApi {

def api_key

def api_secret

def search(query) {

def bearer_token = getBearerToken()

def tweets = []

def api = new HTTPBuilder("https://api.twitter.com/1.1/search/tweets.json?q=${query}&count=100")

api.request(GET,JSON) { req ->

headers."Authorization" = "Bearer " + bearer_token

response.failure = { resp, json ->

println "Unexpected error: ${resp.statusLine.statusCode} : ${resp.statusLine.reasonPhrase}"

resp.headers.each { println "${it.name} : ${it.value}" }

ret = reader.getText()

println ret

}

response.success = { resp, json ->

json.statuses.each {

tweets << [

id: it.id,

text : it.text,

truncated : it.truncated,

createdAt : it.created_at.minus("+0000"),

retweet : it.retweet_count

]

}

return tweets

}

private getBearerToken(){

def encoded_basic = new String(Base64.encodeBase64((api_key+":"+api_secret).bytes))

def api = new HTTPBuilder("https://api.twitter.com")

def access_token

api.request(POST,JSON) { req ->

uri.path = "/oauth2/token"

headers."Content-Type" = "application/x-www-form-urlencoded;charset=UTF-8"

headers."Authorization" = "Basic "+encoded_basic

body = "grant_type=client_credentials"

response.success = { resp, json ->

access_token = json.access_token

}

response.failure = { resp, json ->

println "Unexpected error: ${resp.statusLine.statusCode} : ${resp.statusLine.reasonPhrase}"

resp.headers.each { println "${it.name} : ${it.value}" }

ret = reader.getText()

println ret

}

return access_token

}

endsubmit;

submit parseonly;

import java.util.ArrayList;

import java.util.Iterator;

import java.util.LinkedHashMap;

public class TwitterSAS {

public String api_key = "";

public String api_secret = "";

public String search_query = "";

public void main() {

TwitterApi api = new TwitterApi();

api.setApi_key(api_key);

api.setApi_secret(api_secret);

tweets = api.search(search_query);

iter = tweets.iterator();

}

public boolean hasNext() {

return iter.hasNext();

}

public void getNext() {

tweet = ((LinkedHashMap) (iter.next()));

}

public String getString(String key) {

return tweet.get(key).toString();

}

protected ArrayList tweets;

protected Iterator iter;

protected LinkedHashMap tweet;

}

endsubmit;

quit;

/*-----------------------------------------------------------------------------------------------------------------------------------------

*-Add our PROC GROOVY output to CLASSPATH

options set=classpath "%sysfunc(pathname(cp,f))";

/*-----------------------------------------------------------------------------------------------------------------------------------------

*-Collect Twitter Search Results into SAS DATA SET

data twitter;

length id text truncated createdAt retweet $ 140;

dcl javaobj tweets("TwitterSAS");

tweets.setStringField("api_key" , "&api_key." );

tweets.setStringField("api_secret" , "&api_secret." );

tweets.setStringField("search_query" , "&search_query." );

tweets.callVoidMethod("main");

tweets.callBooleanMethod("hasNext",rc);

do _n_=1 to 100 while(rc);

tweets.callVoidMethod("getNext");

tweets.callStringMethod("getString","id",id);

tweets.callStringMethod("getString","text",text);

tweets.callStringMethod("getString","truncated",truncated);

tweets.callStringMethod("getString","createdAt",createdAt);

tweets.callStringMethod("getString","retweet",retweet);

output;

tweets.callBooleanMethod("hasNext",rc);

end;

run;

Re: Proc Groovy\HTTP Headaches

NandanM — Tue, 15 Jul 2014 03:22:28 GMT

Since it's a jar file, do we need to install a copy of JAVA Runtime Environment to be able to successfully reference it and eventually run the code or is that not required. Currently I don't have any software that can open the jar file

Re: Proc Groovy\HTTP Headaches

NandanM — Mon, 14 Jul 2014 04:15:51 GMT

Below is the proc groovy statement:

proc groovy classpath=cp;

add classpath=ivy;

add classpath="M:\nandanm\groovy-2.3.4.zip";

submit parseonly;

@Grab(group='org.codehaus.groovy.modules.http-builder', module='http-builder', version='0.7')

@Grab(group='commons-codec', module='commons-codec', version='1.2')

import groovy.json.JsonSlurper

import org.apache.commons.codec.binary.Base64

import groovyx.net.http.HTTPBuilder

import static groovyx.net.http.ContentType.JSON

import static groovyx.net.http.Method.GET

import static groovyx.net.http.Method.POST

class TwitterApi {

def api_key

def api_secret

def search(query) {

def bearer_token = getBearerToken()

def tweets = []

def api = new HTTPBuilder("https://api.twitter.com/1.1/search/tweets.json?q=${query}&count=10")

api.request(GET,JSON) { req ->

headers."Authorization" = "Bearer " + bearer_token

response.success = { resp, json ->

json.statuses.each {

tweets << [

id: it.id,

text : it.text,

truncated : it.truncated,

createdAt : it.created_at.minus("+0000"),

userId : it.user.id,

userName : it.user.name,

userScreenName: it.user.screen_name,

location : it.user.location,

description : it.user.description,

url : it.user.url,

userFollowers: it.user.followers_count,

userFriends : it.user.friends_count,

userListed : it.user.listed_count,

retweet : it.retweet_count,

favorite : it.favorite_count

]

}

return tweets

}

private getBearerToken(){

def encoded_basic = new String(Base64.encodeBase64((api_key+":"+api_secret).bytes))

def api = new HTTPBuilder("https://api.twitter.com")

def access_token

api.request(POST,JSON) { req ->

uri.path = "/oauth2/token"

headers."Content-Type" = "application/x-www-form-urlencoded;charset=UTF-8"

headers."Authorization" = "Basic "+encoded_basic

body = "grant_type=client_credentials"

response.success = { resp, json ->

access_token = json.access_token

}

return access_token

}

endsubmit;

submit parseonly;

import java.util.ArrayList;

import java.util.Iterator;

import java.util.LinkedHashMap;

public class TwitterSAS {

public String api_key = "";

public String api_secret = "";

public String search_query = "";

public void main() {

TwitterApi api = new TwitterApi();

api.setApi_key(api_key);

api.setApi_secret(api_secret);

tweets = api.search(search_query);

iter = tweets.iterator();

}

public boolean hasNext() {

return iter.hasNext();

}

public void getNext() {

tweet = ((LinkedHashMap) (iter.next()));

}

public String getString(String key) {

return tweet.get(key).toString();

}

protected ArrayList tweets;

protected Iterator iter;

protected LinkedHashMap tweet;

}

endsubmit;

quit;

The code gives the following error. Below is the Entire log:

1 The SAS System 21:46 Sunday, July 13, 2014

1 ;*';*";*/;quit;run;

2 OPTIONS PAGENO=MIN;

3 %LET _CLIENTTASKLABEL='Program';

4 %LET _CLIENTPROJECTPATH='M:\nandanm\sasGF2014.egp';

5 %LET _CLIENTPROJECTNAME='sasGF2014.egp';

6 %LET _SASPROGRAMFILE=;

8 ODS _ALL_ CLOSE;

9 OPTIONS DEV=ACTIVEX;

10 GOPTIONS XPIXELS=0 YPIXELS=0;

11 FILENAME EGSR TEMP;

12 ODS tagsets.sasreport13(ID=EGSR) FILE=EGSR STYLE=HtmlBlue

12 ! STYLESHEET=(URL="file:///C:/Program%20Files/SASHome/SASEnterpriseGuide/5.1/Styles/HtmlBlue.css") NOGTITLE NOGFOOTNOTE

12 ! GPATH=&sasworklocation ENCODING=UTF8 options(rolap="on");

NOTE: Writing TAGSETS.SASREPORT13(EGSR) Body file: EGSR

14 GOPTIONS ACCESSIBLE;

15 proc

15 ! groovy classpath=cp;

NOTE: The ADD CLASSPATH command completed.

16 ! add classpath=ivy;

NOTE: The ADD CLASSPATH command completed.

17 add classpath="M:\nandanm\groovy-2.3.4.zip";

NOTE: The ADD CLASSPATH command completed.

20 submit parseonly;

23 @Grab(group='org.codehaus.groovy.modules.http-builder', module='http-builder', version='0.7')

24 @Grab(group='commons-codec', module='commons-codec', version='1.2')

27 import groovy.json.JsonSlurper

28 import org.apache.commons.codec.binary.Base64

29 import groovyx.net.http.HTTPBuilder

30 import static groovyx.net.http.ContentType.JSON

31 import static groovyx.net.http.Method.GET

32 import static groovyx.net.http.Method.POST

35 class TwitterApi {

36 def api_key

37 def api_secret

40 def search(query) {

41 def bearer_token = getBearerToken()

42 def tweets = []

45 def api = new HTTPBuilder("https://api.twitter.com/1.1/search/tweets.json?q=${query}&count=10")

48 api.request(GET,JSON) { req ->

49 headers."Authorization" = "Bearer " + bearer_token

2 The SAS System 21:46 Sunday, July 13, 2014

52 response.success = { resp, json ->

53 json.statuses.each {

54 tweets << [

55 id: it.id,

56 text : it.text,

57 truncated : it.truncated,

58 createdAt : it.created_at.minus("+0000"),

59 userId : it.user.id,

60 userName : it.user.name,

61 userScreenName: it.user.screen_name,

62 location : it.user.location,

63 description : it.user.description,

64 url : it.user.url,

65 userFollowers: it.user.followers_count,

66 userFriends : it.user.friends_count,

67 userListed : it.user.listed_count,

68 retweet : it.retweet_count,

69 favorite : it.favorite_count

70 ]

71 }

72 }

73 }

74 return tweets

75 }

78 private getBearerToken(){

79 def encoded_basic = new String(Base64.encodeBase64((api_key+":"+api_secret).bytes))

80 def api = new HTTPBuilder("https://api.twitter.com")

81 def access_token

82 api.request(POST,JSON) { req ->

83 uri.path = "/oauth2/token"

84 headers."Content-Type" = "application/x-www-form-urlencoded;charset=UTF-8"

85 headers."Authorization" = "Basic "+encoded_basic

86 body = "grant_type=client_credentials"

89 response.success = { resp, json ->

90 access_token = json.access_token

91 }

92 }

93 return access_token

94 }

95 }

98 endsubmit;

ERROR: The SUBMIT command failed.

org.codehaus.groovy.control.MultipleCompilationErrorsException: startup failed:

script14053113013111409775131.groovy: 3: unable to resolve class groovy.json.JsonSlurper @ line 3, column 5.

@Grab(group='org.codehaus.groovy.modules.http-builder', module='http-builder', version='0.7')

1 error

at org.codehaus.groovy.control.ErrorCollector.failIfErrors(ErrorCollector.java:296)

at org.codehaus.groovy.control.CompilationUnit.applyToSourceUnits(CompilationUnit.java:860)

3 The SAS System 21:46 Sunday, July 13, 2014

at org.codehaus.groovy.control.CompilationUnit.doPhaseOperation(CompilationUnit.java:521)

at org.codehaus.groovy.control.CompilationUnit.processPhaseOperations(CompilationUnit.java:497)

at org.codehaus.groovy.control.CompilationUnit.compile(CompilationUnit.java:474)

at groovy.lang.GroovyClassLoader.parseClass(GroovyClassLoader.java:292)

at groovy.lang.GroovyClassLoader.parseClass(GroovyClassLoader.java:263)

at groovy.lang.GroovyClassLoader.parseClass(GroovyClassLoader.java:207)

at groovy.lang.GroovyClassLoader.parseClass(GroovyClassLoader.java:217)

100

101 submit parseonly;

102

103

104 import java.util.ArrayList;

105 import java.util.Iterator;

106 import java.util.LinkedHashMap;

107

108

109 public class TwitterSAS {

110 public String api_key = "";

111 public String api_secret = "";

112 public String search_query = "";

113

114 public void main() {

115 TwitterApi api = new TwitterApi();

116 api.setApi_key(api_key);

117 api.setApi_secret(api_secret);

118 tweets = api.search(search_query);

119 iter = tweets.iterator();

120 }

121

122

123 public boolean hasNext() {

124 return iter.hasNext();

125 }

126

127

128 public void getNext() {

129 tweet = ((LinkedHashMap) (iter.next()));

130 }

131

132

133 public String getString(String key) {

134 return tweet.get(key).toString();

135 }

136

137

138 protected ArrayList tweets;

139 protected Iterator iter;

140 protected LinkedHashMap tweet;

141 }

142

143

144 endsubmit;

ERROR: The SUBMIT command failed.

org.codehaus.groovy.control.MultipleCompilationErrorsException: startup failed:

script14053113013901958481143.groovy: 14: unable to resolve class TwitterApi @ line 14, column 24.

TwitterApi api = new TwitterApi();

4 The SAS System 21:46 Sunday, July 13, 2014

script14053113013901958481143.groovy: 14: unable to resolve class TwitterApi @ line 14, column 30.

TwitterApi api = new TwitterApi();

2 errors