1.完成部分索引伪代码
This commit is contained in:
ascrutae
parent
3db7409d5a
commit
8f50a79e69
|
|
@ -1,16 +0,0 @@
|
|||
package com.a.eye.skywalking.storage;
|
||||
|
||||
import org.apache.logging.log4j.LogManager;
|
||||
import org.apache.logging.log4j.Logger;
|
||||
|
||||
public class Main {
|
||||
|
||||
private static final Logger logger = LogManager.getLogger(Main.class);
|
||||
|
||||
public static void main(String[] args) {
|
||||
logger.info("Init config param.");
|
||||
//
|
||||
new DataStorager().start();
|
||||
logger.info("start success.");
|
||||
}
|
||||
}
|
||||
|
|
@ -1,8 +0,0 @@
|
|||
package com.a.eye.skywalking.storage.conf;
|
||||
|
||||
public class Config {
|
||||
|
||||
public static class Index_DB {
|
||||
public static String TIME_RANGE_INDEX_FILE_PATH = "/tmp/skywalking/index";
|
||||
}
|
||||
}
|
||||
|
|
@ -1,71 +0,0 @@
|
|||
package com.a.eye.skywalking.storage.conf;
|
||||
|
||||
import org.apache.logging.log4j.LogManager;
|
||||
import org.apache.logging.log4j.Logger;
|
||||
|
||||
import java.lang.reflect.Field;
|
||||
import java.lang.reflect.Modifier;
|
||||
import java.util.LinkedList;
|
||||
import java.util.Properties;
|
||||
|
||||
public class ConfigInitializer {
|
||||
|
||||
public static Logger logger = LogManager.getLogger(ConfigInitializer.class);
|
||||
|
||||
public static void initialize(Properties properties, Class<?> rootConfigType) throws IllegalAccessException {
|
||||
initNextLevel(properties, rootConfigType, new ConfigDesc());
|
||||
}
|
||||
|
||||
private static void initNextLevel(Properties properties, Class<?> recentConfigType, ConfigDesc parentDesc) throws NumberFormatException, IllegalArgumentException, IllegalAccessException {
|
||||
for (Field field : recentConfigType.getFields()) {
|
||||
if (Modifier.isPublic(field.getModifiers()) && Modifier.isStatic(field.getModifiers())) {
|
||||
String configKey = (parentDesc + "." +
|
||||
field.getName()).toLowerCase();
|
||||
String value = properties.getProperty(configKey);
|
||||
if (value != null) {
|
||||
if (field.getType().equals(int.class))
|
||||
field.set(null, Integer.valueOf(value));
|
||||
if (field.getType().equals(String.class))
|
||||
field.set(null, value);
|
||||
if (field.getType().equals(long.class))
|
||||
field.set(null, Long.valueOf(value));
|
||||
}
|
||||
logger.debug("{}={}", configKey, field.get(null));
|
||||
}
|
||||
}
|
||||
for (Class<?> innerConfiguration : recentConfigType.getClasses()) {
|
||||
parentDesc.append(innerConfiguration.getSimpleName());
|
||||
initNextLevel(properties, innerConfiguration, parentDesc);
|
||||
parentDesc.removeLastDesc();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
class ConfigDesc {
|
||||
private LinkedList<String> descs = new LinkedList<String>();
|
||||
|
||||
void append(String currentDesc) {
|
||||
descs.addLast(currentDesc);
|
||||
}
|
||||
|
||||
void removeLastDesc() {
|
||||
descs.removeLast();
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
if (descs.size() == 0) {
|
||||
return "";
|
||||
}
|
||||
StringBuilder ret = new StringBuilder(descs.getFirst());
|
||||
boolean first = true;
|
||||
for (String desc : descs) {
|
||||
if (first) {
|
||||
first = false;
|
||||
continue;
|
||||
}
|
||||
ret.append(".").append(desc);
|
||||
}
|
||||
return ret.toString();
|
||||
}
|
||||
}
|
||||
|
|
@ -1,41 +0,0 @@
|
|||
package com.a.eye.skywalking.storage.data;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.FileOutputStream;
|
||||
import java.io.IOException;
|
||||
|
||||
/**
|
||||
* 主要功能：
|
||||
* 1. 控制源数据的写入
|
||||
* 2. 当数据文件满足阈值，自动切换文件
|
||||
*/
|
||||
public class DataFileWriter {
|
||||
|
||||
private FileOutputStream outputStream;
|
||||
private File currentOriginDataFile;
|
||||
private String bufferFileBasePath;
|
||||
|
||||
public DataFileWriter(String bufferFileBasePath) {
|
||||
this.bufferFileBasePath = bufferFileBasePath;
|
||||
|
||||
}
|
||||
|
||||
public DataFileWriter(File bufferFile) {
|
||||
|
||||
}
|
||||
|
||||
public int write(byte[] data) throws IOException {
|
||||
outputStream.write(data);
|
||||
return 0;
|
||||
}
|
||||
|
||||
private void convertFile() {
|
||||
|
||||
}
|
||||
|
||||
public void releaseResource() throws IOException {
|
||||
outputStream.flush();
|
||||
outputStream.close();
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,8 @@
|
|||
package com.a.eye.skywalking.storage.data;
|
||||
|
||||
/**
|
||||
* Created by xin on 2016/11/1.
|
||||
*/
|
||||
public class DataOperator {
|
||||
|
||||
}
|
||||
|
|
@ -1,34 +0,0 @@
|
|||
package com.a.eye.skywalking.storage.data;
|
||||
|
||||
import com.a.eye.skywalking.storage.index.IndexData;
|
||||
import org.apache.logging.log4j.LogManager;
|
||||
import org.apache.logging.log4j.Logger;
|
||||
|
||||
import java.io.IOException;
|
||||
|
||||
/**
|
||||
* 主要用于源数据文件的缓存
|
||||
*/
|
||||
public class DataStorager {
|
||||
private static Logger logger = LogManager.getLogger(DataStorager.class);
|
||||
private DataFileWriter writer;
|
||||
|
||||
public void start() {
|
||||
|
||||
}
|
||||
|
||||
public IndexData save(byte[] data) {
|
||||
|
||||
// convert it to data;
|
||||
// IndexData index = new IndexData();
|
||||
|
||||
try {
|
||||
int offset = writer.write(data);
|
||||
} catch (IOException e) {
|
||||
logger.error("Failed to save data");
|
||||
return null;
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,16 @@
|
|||
package com.a.eye.skywalking.storage.data.file;
|
||||
|
||||
/**
|
||||
* Created by xin on 2016/10/31.
|
||||
*/
|
||||
public class DataFileOperator {
|
||||
|
||||
public int write(){
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
public Object read(Object dataIndex) {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,7 @@
|
|||
package com.a.eye.skywalking.storage.data.file;
|
||||
|
||||
/**
|
||||
* Created by xin on 2016/10/31.
|
||||
*/
|
||||
public class DataFileReader {
|
||||
}
|
||||
|
|
@ -0,0 +1,7 @@
|
|||
package com.a.eye.skywalking.storage.data.file;
|
||||
|
||||
/**
|
||||
* Created by xin on 2016/10/31.
|
||||
*/
|
||||
public class DataFileWriter {
|
||||
}
|
||||
|
|
@ -0,0 +1,14 @@
|
|||
package com.a.eye.skywalking.storage.data.index;
|
||||
|
||||
/**
|
||||
* Created by xin on 2016/10/31.
|
||||
*/
|
||||
public class DataIndexFile {
|
||||
|
||||
public void read(String traceId) {
|
||||
}
|
||||
|
||||
public void write() {
|
||||
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,16 @@
|
|||
package com.a.eye.skywalking.storage.data.index;
|
||||
|
||||
/**
|
||||
* Created by xin on 2016/10/31.
|
||||
*/
|
||||
public class DataIndexFileOperator {
|
||||
|
||||
public Object read(String traceId) {
|
||||
return null;
|
||||
}
|
||||
|
||||
public void write(Object o, int offset) {
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,7 @@
|
|||
package com.a.eye.skywalking.storage.data.index;
|
||||
|
||||
/**
|
||||
* Created by xin on 2016/10/31.
|
||||
*/
|
||||
public class DataIndexFileReader {
|
||||
}
|
||||
|
|
@ -0,0 +1,7 @@
|
|||
package com.a.eye.skywalking.storage.data.index;
|
||||
|
||||
/**
|
||||
* Created by xin on 2016/10/31.
|
||||
*/
|
||||
public class DataIndexFileWriter {
|
||||
}
|
||||
|
|
@ -1,11 +0,0 @@
|
|||
package com.a.eye.skywalking.storage.exception;
|
||||
|
||||
/**
|
||||
* Index data cannot be saved.
|
||||
*/
|
||||
public class IndexDataSaveFailedException extends Exception {
|
||||
|
||||
public IndexDataSaveFailedException(String message) {
|
||||
super(message);
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,28 @@
|
|||
package com.a.eye.skywalking.storage.index;
|
||||
|
||||
import com.a.eye.skywalking.storage.index.cache.DataFileIndexCache;
|
||||
|
||||
/**
|
||||
* 用于初始化DataFileIndex的数据，并且提供操作Index的所有类
|
||||
*/
|
||||
public class DataFileIndexEngine {
|
||||
|
||||
private static final DataFileIndexCache indexCache;
|
||||
|
||||
static {
|
||||
|
||||
indexCache = new DataFileIndexCache();
|
||||
}
|
||||
|
||||
private DataFileIndexEngine() {
|
||||
//DO Nothing
|
||||
}
|
||||
|
||||
public static DataFileIndexFinder newFinder() {
|
||||
return indexCache;
|
||||
}
|
||||
|
||||
public static DataFileIndexUpdator newUpdator() {
|
||||
return indexCache;
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,5 @@
|
|||
package com.a.eye.skywalking.storage.index;
|
||||
|
||||
public interface DataFileIndexFinder {
|
||||
DataIndexMetaData find(long timestamp);
|
||||
}
|
||||
|
|
@ -0,0 +1,5 @@
|
|||
package com.a.eye.skywalking.storage.index;
|
||||
|
||||
public interface DataFileIndexUpdator {
|
||||
void update(long timestamp);
|
||||
}
|
||||
|
|
@ -0,0 +1,7 @@
|
|||
package com.a.eye.skywalking.storage.index;
|
||||
|
||||
/**
|
||||
* Created by xin on 2016/11/1.
|
||||
*/
|
||||
public class DataIndexMetaData {
|
||||
}
|
||||
|
|
@ -1,50 +0,0 @@
|
|||
package com.a.eye.skywalking.storage.index;
|
||||
|
||||
/**
|
||||
* Created by xin on 2016/10/31.
|
||||
*/
|
||||
public class TimeRangeOfIndexData {
|
||||
private String indexDataFileName;
|
||||
|
||||
private long startTime;
|
||||
|
||||
private long endTime;
|
||||
|
||||
|
||||
public TimeRangeOfIndexData() {
|
||||
}
|
||||
|
||||
public TimeRangeOfIndexData(String fileName) {
|
||||
this.indexDataFileName = fileName;
|
||||
}
|
||||
|
||||
public TimeRangeOfIndexData(String fileName, long startTime, long endTime) {
|
||||
this.indexDataFileName = fileName;
|
||||
this.startTime = startTime;
|
||||
this.endTime = endTime;
|
||||
}
|
||||
|
||||
|
||||
public String getIndexDataFileName() {
|
||||
return indexDataFileName;
|
||||
}
|
||||
|
||||
public long getStartTime() {
|
||||
return startTime;
|
||||
}
|
||||
|
||||
public long getEndTime() {
|
||||
return endTime;
|
||||
}
|
||||
|
||||
public String buildConnectionURL() {
|
||||
// TODO: 2016/10/31 通过参数构建连接URL
|
||||
return null;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return "TimeRangeOfIndexData{" + "indexDataFileName='" + indexDataFileName + '\'' + ", startTime=" + startTime
|
||||
+ ", endTime=" + endTime + '}';
|
||||
}
|
||||
}
|
||||
|
|
@ -1,62 +0,0 @@
|
|||
package com.a.eye.skywalking.storage.index;
|
||||
|
||||
import com.a.eye.skywalking.storage.conf.Config;
|
||||
import org.apache.logging.log4j.LogManager;
|
||||
import org.apache.logging.log4j.Logger;
|
||||
|
||||
import java.io.*;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
public class TimeRangeOfIndexDataFile {
|
||||
private static Logger logger = LogManager.getLogger(TimeRangeOfIndexDataFile.class);
|
||||
private final String FILE_NAME = "time_range.index";
|
||||
private static final TimeRangeOfIndexDataFile INSTANCE = new TimeRangeOfIndexDataFile();
|
||||
private final File indexFile;
|
||||
|
||||
private TimeRangeOfIndexDataFile() {
|
||||
indexFile = new File(Config.Index_DB.TIME_RANGE_INDEX_FILE_PATH, FILE_NAME);
|
||||
if (!indexFile.exists()) {
|
||||
indexFile.getParentFile().mkdirs();
|
||||
|
||||
try {
|
||||
indexFile.createNewFile();
|
||||
} catch (IOException e) {
|
||||
logger.error("Failed to create time_range.index", e);
|
||||
System.exit(-1);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public void write(TimeRangeOfIndexData timeRange) {
|
||||
try {
|
||||
FileWriter writer = new FileWriter(indexFile);
|
||||
writer.write(
|
||||
timeRange.getIndexDataFileName() + "\t" + timeRange.getStartTime() + "\t" + timeRange.getEndTime());
|
||||
} catch (IOException e) {
|
||||
logger.error("Failed to write {} to index file", timeRange, e);
|
||||
}
|
||||
}
|
||||
|
||||
public List<TimeRangeOfIndexData> read() {
|
||||
List<TimeRangeOfIndexData> indexData = new ArrayList<TimeRangeOfIndexData>();
|
||||
try {
|
||||
BufferedReader reader = new BufferedReader(new FileReader(indexFile));
|
||||
String indexDataStr = null;
|
||||
while ((reader.readLine()) != null) {
|
||||
String[] indexSegment = indexDataStr.split("\t");
|
||||
indexData.add(new TimeRangeOfIndexData(indexSegment[0], Long.parseLong(indexSegment[1]),
|
||||
Long.parseLong(indexSegment[2])));
|
||||
}
|
||||
} catch (IOException e) {
|
||||
logger.error("Failed to read data from index file", e);
|
||||
}
|
||||
|
||||
return indexData;
|
||||
}
|
||||
|
||||
|
||||
public static TimeRangeOfIndexDataFile INSTANCE() {
|
||||
return INSTANCE;
|
||||
}
|
||||
}
|
||||
|
|
@ -1,132 +0,0 @@
|
|||
package com.a.eye.skywalking.storage.index;
|
||||
|
||||
import com.zaxxer.hikari.HikariConfig;
|
||||
import com.zaxxer.hikari.HikariDataSource;
|
||||
import org.apache.logging.log4j.LogManager;
|
||||
import org.apache.logging.log4j.Logger;
|
||||
|
||||
import java.sql.Connection;
|
||||
import java.sql.PreparedStatement;
|
||||
import java.sql.ResultSet;
|
||||
import java.sql.SQLException;
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* Created by xin on 2016/10/31.
|
||||
*/
|
||||
public class TimeRangeOfIndexDataFinder {
|
||||
private static Logger logger = LogManager.getLogger(TimeRangeOfIndexDataFinder.class);
|
||||
private static final String SQL_JDBC_URL = "jdbc:hsqldb:mem:TIME_RANGE";
|
||||
private static final String SQL_CREATE_TABLE =
|
||||
"CREATE TABLE Time_Range("
|
||||
+ "id INT PRIMARY KEY NOT NULL IDENTITY,"
|
||||
+ "fileName VARCHAR(32) NOT NULL," + " "
|
||||
+ "startTime INT NOT NULL,"
|
||||
+ "endTime INT NOT NULL\n"
|
||||
+ ");";
|
||||
private static final String SQL_CREATE_INDEX =
|
||||
"CREATE INDEX \"Time_Range_startTime_endTime_index\" ON Time_Range (startTime, endTime);";
|
||||
private static final String SQL_INSERT_DATA = "INSERT INTO Time_Range(fileName,startTime,endTime) VALUES(?,?,?);";
|
||||
private static final String SQL_SELECT_DATA = "SELECT fileName FROM Time_Range WHERE startTime < ? AND endTime >?";
|
||||
|
||||
|
||||
|
||||
private static final TimeRangeOfIndexDataFinder finder = new TimeRangeOfIndexDataFinder();
|
||||
|
||||
private HikariDataSource hikariDataSource;
|
||||
|
||||
private TimeRangeOfIndexDataFinder() {
|
||||
initDataSource();
|
||||
initTableAndIndex();
|
||||
initTimeRangeData();
|
||||
}
|
||||
|
||||
private void initDataSource() {
|
||||
HikariConfig config = new HikariConfig();
|
||||
config.setJdbcUrl(SQL_JDBC_URL);
|
||||
// TODO: 2016/10/31 初始化数据源参数
|
||||
hikariDataSource = new HikariDataSource(config);
|
||||
}
|
||||
|
||||
private void initTableAndIndex() {
|
||||
Connection connection = null;
|
||||
try {
|
||||
connection = hikariDataSource.getConnection();
|
||||
PreparedStatement ps = connection.prepareStatement(SQL_CREATE_TABLE);
|
||||
ps.execute();
|
||||
|
||||
ps = connection.prepareStatement(SQL_CREATE_INDEX);
|
||||
ps.execute();
|
||||
} catch (SQLException e) {
|
||||
logger.error("Failed to ");
|
||||
} finally {
|
||||
if (connection != null) {
|
||||
try {
|
||||
connection.close();
|
||||
} catch (SQLException e) {
|
||||
logger.error("Failed to close connection when init time range data");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void initTimeRangeData() {
|
||||
List<TimeRangeOfIndexData> timeRangeOfIndexDataList = TimeRangeOfIndexDataFile.INSTANCE().read();
|
||||
Connection connection = null;
|
||||
int currentCount = 0;
|
||||
try {
|
||||
connection = hikariDataSource.getConnection();
|
||||
PreparedStatement ps = connection.prepareStatement(SQL_INSERT_DATA);
|
||||
for (TimeRangeOfIndexData data : timeRangeOfIndexDataList) {
|
||||
ps.setString(1, data.getIndexDataFileName());
|
||||
ps.setLong(2, data.getStartTime());
|
||||
ps.setLong(3, data.getEndTime());
|
||||
|
||||
if ((currentCount++) % 100 == 0) {
|
||||
ps.executeBatch();
|
||||
}
|
||||
}
|
||||
|
||||
ps.executeBatch();
|
||||
} catch (SQLException e) {
|
||||
logger.error("Failed to init time range data");
|
||||
} finally {
|
||||
if (connection != null) {
|
||||
try {
|
||||
connection.close();
|
||||
} catch (SQLException e) {
|
||||
logger.error("Failed to close connection when init time range data");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
||||
public static TimeRangeOfIndexDataFinder INSTANCE() {
|
||||
return finder;
|
||||
}
|
||||
|
||||
public TimeRangeOfIndexData find(long timestamp) {
|
||||
Connection connection = null;
|
||||
try {
|
||||
connection = hikariDataSource.getConnection();
|
||||
PreparedStatement ps = connection.prepareStatement(SQL_SELECT_DATA);
|
||||
ps.setLong(1, timestamp);
|
||||
ps.setLong(2, timestamp);
|
||||
ResultSet rs = ps.executeQuery();
|
||||
rs.next();
|
||||
return new TimeRangeOfIndexData(rs.getString("fileName"));
|
||||
} catch (SQLException e) {
|
||||
return null;
|
||||
} finally {
|
||||
if (connection != null) {
|
||||
try {
|
||||
connection.close();
|
||||
} catch (SQLException e) {
|
||||
logger.error("Failed to close connection when find data.");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,26 @@
|
|||
package com.a.eye.skywalking.storage.index.cache;
|
||||
|
||||
import com.a.eye.skywalking.storage.index.DataFileIndexFinder;
|
||||
import com.a.eye.skywalking.storage.index.DataFileIndexUpdator;
|
||||
import com.a.eye.skywalking.storage.index.DataIndexMetaData;
|
||||
|
||||
public class DataFileIndexCache implements DataFileIndexFinder, DataFileIndexUpdator {
|
||||
|
||||
private IndexL1Cache l1Cache;
|
||||
private IndexL2Cache l2Cache;
|
||||
|
||||
public DataFileIndexCache() {
|
||||
//load the index data
|
||||
}
|
||||
|
||||
@Override
|
||||
public void update(long timestamp) {
|
||||
// TODO: 2016/11/1 通知Cache更新，以及Index的文件更新
|
||||
}
|
||||
|
||||
@Override
|
||||
public DataIndexMetaData find(long timestamp) {
|
||||
// TODO: 2016/11/1 查找一级缓存，二级缓存
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,9 @@
|
|||
package com.a.eye.skywalking.storage.index.cache;
|
||||
|
||||
import java.util.SortedSet;
|
||||
|
||||
public class DataFileIndexReader {
|
||||
public SortedSet<Long> read() {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,8 @@
|
|||
package com.a.eye.skywalking.storage.index.cache;
|
||||
|
||||
public class DataFileIndexWriter {
|
||||
|
||||
public void write(long timestamp) {
|
||||
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,7 @@
|
|||
package com.a.eye.skywalking.storage.index.cache;
|
||||
|
||||
/**
|
||||
* Created by xin on 2016/11/1.
|
||||
*/
|
||||
class IndexL1Cache {
|
||||
}
|
||||
|
|
@ -0,0 +1,7 @@
|
|||
package com.a.eye.skywalking.storage.index.cache;
|
||||
|
||||
/**
|
||||
* Created by xin on 2016/11/1.
|
||||
*/
|
||||
class IndexL2Cache {
|
||||
}
|
||||
Loading…
Reference in New Issue