. Chan Y. "A Distributed Stream Library for Java 8," Ph.D. dissertation, University of York, 2016.
. Chowdhury M., Zaharia M., Stoica I., Performance and Scalability of Broadcast in Spark. 2010.
. Dewire D.T. Client-server computing. McGrawHill, Singapore, 1993.
. Digital Fuel of the 21st Century: Innovation through Open Data and the Network Effect, Harvard University, 2011.
. Eur-Lex, Open data An engine for innovation, growth and transparent governance, 2011.
. Google Maps Documentation [Электронный ресурс] // URL:https://developers.google.com/maps/documentation (Дата обращения 02.03.2017).
8. Grosso W. Java RMI. First Edition. O'Reilly and Associates, USA, 2001.
. Kumar K., Liu J., Lu Y., Bhargava B., "A survey of computation offloading for mobile systems," Mobile Networks and Applications, vol. 18, no. 1, pp. 129-140, 2013.
. Lahiri M., Berger-Wolf T., "Mining Periodic Behavior in Dynamic Social Networks," Proceedings of the 8th IEEE Inter-national Conference on Data Mining, 2008, pp. 373-382.
. Microservices [Электронный ресурс] // URL: http://microservices.io/patterns/microservices.html (Дата обращения 20.04.2017).
12. O’Reilly T., What Is Web 2.0 [Электронный ресурс] // URL: http://www.oreillynet.com/pub/a/oreilly/tim/news/2005/09/30/what-is-web-20.html (Дата обращения 16.02.2017).
13. Open Data Incubator Europe [Электронный ресурс] // URL: https://opendataincubator.eu/ (Дата обращения 16.02.2017).
14. OpenData 500 [Электронный ресурс] // URL: http://www.opendata500.com/ (Дата обращения 16.02.2017).
15. OpenStreetMap [Электронный ресурс] // URL: https://www.openstreetmap.org/ (Дата обращения 02.03.2017).
16. Ricardo T., Marco T.V., Roberto S.B. An approach for extracting modules from monolithic software architectures. Workshop, pages 1-18, 2012.
. Scharl, A. Towards the Geospatial Web: Media Platforms for Managing Geotagged Knowledge Repositories. The Geospatial Web Geobrowsers, Social Software and the Web 2.0 are Shaping the Network Society (pp. 3-14). London: Springer, 2007.
. Twitter Developer Documentation [Электронный ресурс] // URL: https://dev.twitter.com/overview/api/tweets (Дата обращения 18.04.2017).
. Venner J., Pro Hadoop. Apress, June 22, 2009.
. Wang G., Xiong Y., Yun J., Cavallaro J. "Accelerating computer vision algorithms using OpenCL framework on the mobile GPU-a case study," in Proceedings of the 38th IEEE International Conference on Acoustics, Speech, and Signal Processing (ICASSP ’13), IEEE, Vancouver, Canada, May 2013.
21. Бегтин И.В. Проблема открытых данных в России // Земля из космоса. 2011. №11. С.20-25.
22. Борисенко О.Д., Турдаков Д.Ю., Кузнецов С.Д., Автоматическое создание виртуальных кластеров Apache Spark в облачной среде OpenStack. Труды Института системного программирования РАН, том 17, 2009 г. Стр 31-50.
23. Катков Е.В., Сорочайкин А.Н. Моедирование процессов инновационного развития предприятий // Вестник Самарского государственного университета. 2012. №10. С.33-39.
24. Остервальдер А., Пинье И. Построение бизнес-моделей. Настольная книга стратега и новатора. - М.: Альпина Паблишер, 2011.
. Открытые государственные финансовые данные [Электронный ресурс] // URL: http://budgetapps.ru (Дата обращения 16.02.2017).
. Парамонов В., ГОС: Объем информации в интернете удваивается каждые полтора года [Электронный ресурс] // URL: http://www.newsland.ru/news/detail/id/367158/ (Дата обращения 15.02.2017).
. Портал открытых данных правительства Москвы [Электронный ресурс] // URL: https://data.mos.ru/ (Дата обращения 18.04.2017).
. Портал открытых данных Российской Федерации [Электронный ресурс] // URL: http://data.gov.ru/ (Дата обращения 18.04.2017).
. Хакатон портала открытых данных data.gov.ru [Электронный ресурс] // URL: http://data.gov.ru/hackathon (Дата обращения 16.02.2017).
. Число пользователей Facebook превысило 1,71 миллиарда [Электронный ресурс] // URL: https://rg.ru/2016/07/28/chislo-polzovatelej-facebook-prevysilo-171-milliarda.html (Дата обращения 16.02.2017).
. Что такое открытые данные? [Электронный ресурс] // URL: https://opengovdata.ru/definition/ (Дата обращения 16.02.2017).
. Яндекс. API карт. Документация. [Электронный ресурс] // URL: https://tech.yandex.ru/maps/doc/jsapi/2.1/quick-start/tasks/quick-start-docpage/ (Дата обращения 02.03.2017).
Приложение
Текст программы
MapController.java
package com.alexcodes.web.controller;
import com.alexcodes.web.dto.MapDTO;com.alexcodes.web.service.MapService;org.springframework.beans.factory.annotation.Autowired;org.springframework.http.MediaType;org.springframework.util.Assert;org.springframework.web.bind.annotation.RequestMapping;org.springframework.web.bind.annotation.RequestMethod;org.springframework.web.bind.annotation.RestController;
@RequestMapping("/map")class MapController {final MapService mapService;
@AutowiredMapController(MapService mapService) {.notNull(mapService, "Cannot be null");.mapService = mapService;
}
@RequestMapping(value = "/coordinates",= RequestMethod.GET,= MediaType.APPLICATION_JSON_VALUE)MapDTO getCoordinates() {mapService.getMap();
}
}
CoordinatesService.java
package com.alexcodes.web.service;java.util.List;interface CoordinatesService {<List<Double" findCoordinates();
}
MapService.java
package com.alexcodes.web.service;com.alexcodes.web.dto.MapDTO;org.springframework.beans.factory.annotation.Autowired;org.springframework.stereotype.Service;org.springframework.util.Assert;
@Serviceclass MapService {final CoordinatesService coordinatesService;
@AutowiredMapService(CoordinatesService coordinatesService) {.notNull(coordinatesService, "Cannot be null");.coordinatesService = coordinatesService;
}MapDTO getMap() {dto = new MapDTO();.coordinates = coordinatesService.findCoordinates();dto;
}
}
SimpleCoordinatesService.java
package com.alexcodes.web.service;com.alexcodes.common.dao.GeoPostRepository;com.alexcodes.common.domain.GeoPost;com.google.common.collect.Lists;org.springframework.beans.factory.annotation.Autowired;org.springframework.context.annotation.Profile;org.springframework.stereotype.Service;java.util.Arrays;java.util.List;java.util.Objects;java.util.stream.Collectors;
@Service
@Profile("default")class SimpleCoordinatesService implements CoordinatesService {final GeoPostRepository geoPostRepository;
@AutowiredSimpleCoordinatesService(GeoPostRepository geoPostRepository) {.geoPostRepository = geoPostRepository;
}
@OverrideList<List<Double" findCoordinates() {<GeoPost> posts = Lists.newArrayList(geoPostRepository.findAll());posts.stream()
.map(post -> post.location)
.filter(Objects::nonNull)
.filter(location -> location.latitude != 55.7547875 && location.longitude != 37.427642500000005)
.map(location -> Arrays.asList(location.latitude, location.longitude))
.collect(Collectors.toList());
}
}
CoordinateDTO.java
package com.alexcodes.web.dto;class CoordinateDTO {double longitude;double latitude;CoordinateDTO() {
}CoordinateDTO(double longitude, double latitude) {.longitude = longitude;.latitude = latitude;
}
}
MapDTO.java
package com.alexcodes.web.dto;java.time.Instant;java.util.List;class MapDTO {Instant lastModified;List<List<Double" coordinates;
}
WebMain.java
package com.alexcodes.web;org.springframework.boot.SpringApplication;org.springframework.boot.autoconfigure.SpringBootApplication;org.springframework.context.annotation.ComponentScan;
@SpringBootApplication
@ComponentScan(basePackages = "com.alexcodes.*")class WebMain {static void main(String[] args) {.run(WebMain.class);
}
}
Index.html
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1, shrink-to-fit=no">
<title>Social Parks</title>
<link href="https://yandex.st/bootstrap/2.3.2/css/bootstrap.min.css" rel="stylesheet">
<style type="text/css">, body, .hero-unit {height: 100%;: 100%;: 0;
}
#YMapsID {: 900px;: 700px;
}
#YMapsCode {: 880px;
}
</style>
<script src="js/jquery-3.2.1.min.js" type="text/javascript"></script>
<script src="http://api-maps.yandex.ru/2.1/?lang=ru_RU" type="text/javascript"></script>
<script src="js/heatmap.min.js" type="text/javascript"></script>
<script type="text/javascript">.ready(function () {
$.get("/map/coordinates", function (response) {data = response.coordinates;map = new ymaps.Map('YMapsID', {: [55.751588, 37.617861],: ['zoomControl', 'typeSelector', 'fullscreenControl'],: 11, type: 'yandex#satellite'
}),= {: new ymaps.control.Button({: {: 'Toggle dissipating'
},: {: false,: 150
}
}),: new ymaps.control.Button({: {: 'Change opacity'
},: {: false,: 150
}
},: {: false,: 150
}
}),: new ymaps.control.Button({: {: 'Reverse gradient'
},: {: false,: 150
}
}),: new ymaps.control.Button({: {: 'Toggle Heatmap'
},: {: false,: 150
}
})
},= [{
.1: 'rgba(128, 255, 0, 0.7)',
.2: 'rgba(255, 255, 0, 0.8)',
.7: 'rgba(234, 72, 58, 0.9)',
.0: 'rgba(162, 36, 25, 1)'
}, {
.1: 'rgba(162, 36, 25, 0.7)',
.2: 'rgba(234, 72, 58, 0.8)',
.7: 'rgba(255, 255, 0, 0.9)',
.0: 'rgba(128, 255, 0, 1)'
}],= [5, 10, 20, 30],= [0.4, 0.6, 0.8, 1];.modules.require(['Heatmap'], function (Heatmap) {heatmap = new Heatmap(data, {: gradients[0],: radiuses[1],: opacities[2]
});.setMap(map);.dissipating.events.add('press', function () {.options.set(
'dissipating', !heatmap.options.get('dissipating')
);
});.opacity.events.add('press', function () {current = heatmap.options.get('opacity'),= opacities.indexOf(current);.options.set(
'opacity', opacities[++index == opacities.length ? 0 : index]
);
});.radius.events.add('press', function () {current = heatmap.options.get('radius'),= radiuses.indexOf(current);.options.set(
'radius', radiuses[++index == radiuses.length ? 0 : index]
);
});.gradient.events.add('press', function () {current = heatmap.options.get('gradient');.options.set(
'gradient', current == gradients[0] ? gradients[1] : gradients[0]
);
});.heatmap.events.add('press', function () {.setMap(.getMap() ? null : map
);
});(var key in buttons) {(buttons.hasOwnProperty(key)) {.controls.add(buttons[key]);
}
}
});
});
});
</script>
</head>
<body>
<div>
<div>
<p>Yandex Maps API <a href="https://github.com/yandex/mapsapi-heatmap">Heatmap Module</a></p>
<div id="YMapsID"></div>
</div>
</div>
</body>
</html>
MongoConfiguration.java
package com.alexcodes.common.config;com.mongodb.Mongo;com.mongodb.MongoClient;org.springframework.beans.factory.annotation.Value;org.springframework.context.annotation.Configuration;org.springframework.data.mongodb.config.AbstractMongoConfiguration;org.springframework.data.mongodb.repository.config.EnableMongoRepositories;
@Configuration
@EnableMongoRepositories({"com.alexcodes.common.dao"})class MongoConfiguration extends AbstractMongoConfiguration {
@Value("${spring.data.mongodb.host}")String host;
@Value("${spring.data.mongodb.port}")int port;
@Value("${spring.data.mongodb.database}")String database;
@OverrideString getDatabaseName() {database;
}
@OverrideMongo mongo() throws Exception {new MongoClient(host, port);
}
}
GeoPost.java
package com.alexcodes.common.domain;com.google.common.base.MoreObjects;org.springframework.data.annotation.Id;java.time.Instant;class GeoPost {
@IdString id;SourceType sourceType;String text;Location location;Instant timestamp;
@OverrideString toString() {MoreObjects.toStringHelper(this)
.add("sourceType", sourceType)
.add("text", text)
.add("location", location)
.toString();
}
}
Location.java
package com.alexcodes.common.domain;com.google.common.base.MoreObjects;class Location {Type type;double longitude;double latitude;Location() {}Location(Type type, double longitude, double latitude) {.type = type;.longitude = longitude;.latitude = latitude;
}
@OverrideString toString() {MoreObjects.toStringHelper(this)
.add("longitude", longitude)
.add("latitude", latitude)
.toString();
}enum Type {, CITY
}
}
AppConfig.java
package com.alexcodes.twitter.config;com.twitter.hbc.core.endpoint.Location;org.springframework.beans.factory.annotation.Value;org.springframework.context.annotation.Bean;org.springframework.context.annotation.Configuration;
@Configurationclass TwitterConfig {
@Value("${twitter.location.southwest.longitude}")double southWestLongitude;
@Value("${twitter.location.southwest.latitude}")double southWestLatitude;
@Value("${twitter.location.northeast.longitude}")double northEastLongitude;
@Value("${twitter.location.northeast.latitude}")double northEastLatitude;
@BeanLocation location() {new Location(Location.Coordinate(southWestLongitude, southWestLatitude),Location.Coordinate(northEastLongitude, northEastLatitude));
}
}
LocationExtractor.java
package com.alexcodes.twitter.logic;com.alexcodes.common.domain.Location;com.google.gson.JsonArray;com.google.gson.JsonElement;com.google.gson.JsonObject;org.slf4j.Logger;org.slf4j.LoggerFactory;org.springframework.stereotype.Service;java.util.ArrayList;java.util.List;
@Serviceclass LocationExtractor {static final Logger log = LoggerFactory.getLogger(LocationExtractor.class);static final String COORDINATES = "coordinates";static final String PLACE = "place";static final String PLACE_TYPE = "place_type";static final String CITY = "city";static final String BOUNDING_BOX = "bounding_box";static final String TYPE = "type";static final String POLYGON = "Polygon";Location getLocation(JsonObject root) {coordinates = root.get(COORDINATES);(coordinates != null && !coordinates.isJsonNull())extractCoordinates(coordinates);place = root.get(PLACE);(place != null && !place.isJsonNull())extractPlace(place);.error("Cannot extract location from {}", root);null;
}Location extractCoordinates(JsonElement coordinates) {array = coordinates.getAsJsonObject().getAsJsonArray(COORDINATES);longitude = array.get(0).getAsDouble();latitude = array.get(1).getAsDouble();new Location(Location.Type.POINT, longitude, latitude);
}Location extractPlace(JsonElement place) {placeType = place.getAsJsonObject().get(PLACE_TYPE).getAsString();boundingBox = place.getAsJsonObject().getAsJsonObject(BOUNDING_BOX);(!placeType.equals(CITY)) {.warn("Unknown placeType: {}", placeType);null;
}point = extractBoundingBox(boundingBox);new Location(Location.Type.CITY, point.longitude, point.latitude);
}Point extractBoundingBox(JsonObject boundingBox) {type = boundingBox.getAsJsonPrimitive(TYPE).getAsString();(type) {POLYGON: {coordinates = boundingBox.getAsJsonArray(COORDINATES)
.get(0)
.getAsJsonArray();<Double> longs = new ArrayList<>(coordinates.size());<Double> lats = new ArrayList<>(coordinates.size());(int i = 0; i < coordinates.size(); i++) {point = coordinates.get(i).getAsJsonArray();.add(point.get(0).getAsDouble());.add(point.get(1).getAsDouble());
}avgLong = longs.stream()
.mapToDouble(Double::doubleValue)
.average()
.getAsDouble();avgLat = lats.stream()
.mapToDouble(Double::doubleValue)
.average()
.getAsDouble();new Point(avgLong, avgLat);
}:.error("Unknown format: {}", boundingBox);new Point(0.0, 0.0);
}
}static class Point {double longitude;double latitude;Point(double longitude, double latitude) {.longitude = longitude;.latitude = latitude;
}
}
}
PostConverter.java
package com.alexcodes.twitter.logic;com.alexcodes.common.domain.GeoPost;com.alexcodes.common.domain.SourceType;com.alexcodes.common.logic.PostConverter;com.google.gson.JsonElement;com.google.gson.JsonObject;com.google.gson.JsonParser;org.springframework.beans.factory.annotation.Autowired;org.springframework.stereotype.Service;java.time.Instant;java.time.format.DateTimeFormatter;
@Serviceclass TweetConverter implements PostConverter {static final String TEXT = "text";static final String CREATED_AT = "created_at";static final DateTimeFormatter formatter =.ofPattern("E MMM dd HH:mm:ss Z yyyy");final LocationExtractor locationExtractor;
@AutowiredTweetConverter(LocationExtractor locationExtractor) {.locationExtractor = locationExtractor;
}
@OverrideGeoPost convert(String message) {parser = new JsonParser();root = parser.parse(message).getAsJsonObject();post = new GeoPost();.sourceType = SourceType.TWITTER;.text = getText(root);.location = locationExtractor.getLocation(root);.timestamp = getTimestamp(root);post;
}String getText(JsonObject root) {root.getAsJsonPrimitive(TEXT).getAsString();
}Instant getTimestamp(JsonObject root) {createdAt = root.get(CREATED_AT);(createdAt == null) return null;timestamp = createdAt.getAsString();formatter.parse(timestamp).query(Instant::from);
}
}
PostProcessor.java
package com.alexcodes.twitter.logic;com.alexcodes.common.dao.GeoPostRepository;com.alexcodes.common.domain.GeoPost;com.alexcodes.common.logic.PostProcessor;org.slf4j.Logger;org.slf4j.LoggerFactory;org.springframework.beans.factory.annotation.Autowired;org.springframework.stereotype.Service;
@Serviceclass TweetProcessor implements PostProcessor {static final Logger log = LoggerFactory.getLogger(TweetProcessor.class);final TweetConverter tweetConverter;final GeoPostRepository geoPostRepository;
@AutowiredTweetProcessor(tweetConverter,geoPostRepository) {.tweetConverter = tweetConverter;.geoPostRepository = geoPostRepository;
}
@Overridevoid process(String message) {{post = tweetConverter.convert(message);.debug("Tweet: {}", post);.save(post);
} catch (RuntimeException e) {.error("Exception during tweet processing:", e);
}
}
}
Listener.java
package com.alexcodes.twitter.service;com.alexcodes.twitter.logic.TweetProcessor;com.twitter.hbc.ClientBuilder;com.twitter.hbc.core.Client;com.twitter.hbc.core.Constants;com.twitter.hbc.core.Hosts;com.twitter.hbc.core.HttpHosts;com.twitter.hbc.core.endpoint.Location;com.twitter.hbc.core.endpoint.StatusesFilterEndpoint;com.twitter.hbc.core.event.Event;com.twitter.hbc.core.processor.StringDelimitedProcessor;com.twitter.hbc.httpclient.auth.Authentication;com.twitter.hbc.httpclient.auth.OAuth1;org.slf4j.Logger;org.slf4j.LoggerFactory;org.springframework.beans.factory.annotation.Autowired;org.springframework.beans.factory.annotation.Value;org.springframework.boot.CommandLineRunner;org.springframework.stereotype.Service;org.springframework.util.Assert;java.util.Collections;java.util.List;java.util.concurrent.BlockingQueue;java.util.concurrent.LinkedBlockingQueue;
@Serviceclass TwitterListener implements CommandLineRunner {static final Logger log = LoggerFactory.getLogger(TwitterListener.class);static final int QUEUE_SIZE = 1000;
@Value("${twitter.consumerKey}")String consumerKey;
@Value("${twitter.consumerSecret}")String consumerSecret;
@Value("${twitter.tokenSecret}")String tokenSecret;
@Value("${twitter.client.name}")String clientName;final Location location;final TweetProcessor tweetProcessor;
@AutowiredTwitterListener(Location location, TweetProcessor tweetProcessor) {.notNull(location, "Location cannot be null");.notNull(tweetProcessor, "Cannot be null");.location = location;.tweetProcessor = tweetProcessor;
}void run(String... strings) throws Exception {
// Set up your blocking queues: Be sure to size these properly based on expected TPS of your stream<String> msgQueue = new LinkedBlockingQueue<>(QUEUE_SIZE);<Event> eventQueue = new LinkedBlockingQueue<>(QUEUE_SIZE);
// Declare the host you want to connect to, the endpoint, and authentication (basic auth or oauth)hosebirdHosts = new HttpHosts(Constants.STREAM_HOST);hosebirdEndpoint = new StatusesFilterEndpoint();
// Optional: set up some followings and track terms<Location> locations = Collections.singletonList(location);.locations(locations);
// These secrets should be read from a config filehosebirdAuth = new OAuth1(consumerKey, consumerSecret, token, tokenSecret);builder = new ClientBuilder()
.name(clientName)
.hosts(hosebirdHosts)
.authentication(hosebirdAuth)
.endpoint(hosebirdEndpoint)
.processor(new StringDelimitedProcessor(msgQueue))
.eventMessageQueue(eventQueue);hosebirdClient = builder.build();.connect();i = 0;time = System.currentTimeMillis();
// on a different thread, or multiple different threads....(!hosebirdClient.isDone()) {message = msgQueue.take();
// log.debug("Receive: {}", message);.process(message);++;
// if (System.currentTimeMillis() - time > 60_000L) break;
}.info("{} TPS", i);.stop();
}
}