full-producer.cpp
Go to the documentation of this file.
1 /* -*- Mode:C++; c-file-style:"gnu"; indent-tabs-mode:nil; -*- */
2 /*
3  * Copyright (c) 2014-2020, The University of Memphis
4  *
5  * This file is part of PSync.
6  * See AUTHORS.md for complete list of PSync authors and contributors.
7  *
8  * PSync is free software: you can redistribute it and/or modify it under the terms
9  * of the GNU Lesser General Public License as published by the Free Software Foundation,
10  * either version 3 of the License, or (at your option) any later version.
11  *
12  * PSync is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY;
13  * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
14  * PURPOSE. See the GNU Lesser General Public License for more details.
15  *
16  * You should have received a copy of the GNU Lesser General Public License along with
17  * PSync, e.g., in COPYING.md file. If not, see <http://www.gnu.org/licenses/>.
18  */
19 
20 #include "PSync/full-producer.hpp"
21 #include "PSync/detail/state.hpp"
22 #include "PSync/detail/util.hpp"
23 
24 #include <ndn-cxx/security/validator-null.hpp>
25 #include <ndn-cxx/util/logger.hpp>
26 #include <ndn-cxx/util/segment-fetcher.hpp>
27 
28 #include <cstring>
29 
30 namespace psync {
31 
33 
34 FullProducer::FullProducer(const size_t expectedNumEntries,
35  ndn::Face& face,
36  const ndn::Name& syncPrefix,
37  const ndn::Name& userPrefix,
38  const UpdateCallback& onUpdateCallBack,
39  ndn::time::milliseconds syncInterestLifetime,
40  ndn::time::milliseconds syncReplyFreshness,
41  CompressionScheme ibltCompression,
42  CompressionScheme contentCompression)
43  : ProducerBase(expectedNumEntries, face, syncPrefix, userPrefix, syncReplyFreshness,
44  ibltCompression, contentCompression)
45  , m_syncInterestLifetime(syncInterestLifetime)
46  , m_onUpdate(onUpdateCallBack)
47  , m_jitter(100, 500)
48 {
49  m_registeredPrefix = m_face.setInterestFilter(
50  ndn::InterestFilter(m_syncPrefix).allowLoopback(false),
51  std::bind(&FullProducer::onSyncInterest, this, _1, _2),
52  std::bind(&FullProducer::onRegisterFailed, this, _1, _2));
53 
54  // Should we do this after setInterestFilter success call back
55  // (Currently following ChronoSync's way)
56  sendSyncInterest();
57 }
58 
60 {
61  if (m_fetcher) {
62  m_fetcher->stop();
63  }
64 }
65 
66 void
67 FullProducer::publishName(const ndn::Name& prefix, ndn::optional<uint64_t> seq)
68 {
69  if (m_prefixes.find(prefix) == m_prefixes.end()) {
70  NDN_LOG_WARN("Prefix not added: " << prefix);
71  return;
72  }
73 
74  uint64_t newSeq = seq.value_or(m_prefixes[prefix] + 1);
75 
76  NDN_LOG_INFO("Publish: " << prefix << "/" << newSeq);
77 
78  updateSeqNo(prefix, newSeq);
79 
80  satisfyPendingInterests();
81 }
82 
83 void
84 FullProducer::sendSyncInterest()
85 {
86  // If we send two sync interest one after the other
87  // since there is no new data in the network yet,
88  // when data is available it may satisfy both of them
89  if (m_fetcher) {
90  m_fetcher->stop();
91  }
92 
93  // Sync Interest format for full sync: /<sync-prefix>/<ourLatestIBF>
94  ndn::Name syncInterestName = m_syncPrefix;
95 
96  // Append our latest IBF
97  m_iblt.appendToName(syncInterestName);
98 
99  m_outstandingInterestName = syncInterestName;
100 
101  m_scheduledSyncInterestId =
102  m_scheduler.schedule(m_syncInterestLifetime / 2 + ndn::time::milliseconds(m_jitter(m_rng)),
103  [this] { sendSyncInterest(); });
104 
105  ndn::Interest syncInterest(syncInterestName);
106 
107  using ndn::util::SegmentFetcher;
108  SegmentFetcher::Options options;
109  options.interestLifetime = m_syncInterestLifetime;
110  options.maxTimeout = m_syncInterestLifetime;
111  options.rttOptions.initialRto = m_syncInterestLifetime;
112 
113  m_fetcher = SegmentFetcher::start(m_face, syncInterest,
114  ndn::security::getAcceptAllValidator(), options);
115 
116  m_fetcher->onComplete.connect([this, syncInterest] (const ndn::ConstBufferPtr& bufferPtr) {
117  onSyncData(syncInterest, bufferPtr);
118  });
119 
120  m_fetcher->onError.connect([this] (uint32_t errorCode, const std::string& msg) {
121  NDN_LOG_ERROR("Cannot fetch sync data, error: " << errorCode << ", message: " << msg);
122  // We would like to recover from errors like NoRoute NACK quicker than sync Interest timeout.
123  // We don't react to Interest timeout here as we have scheduled the next sync Interest
124  // to be sent in half the sync Interest lifetime + jitter above. So we would react to
125  // timeout before it happens.
126  if (errorCode != SegmentFetcher::ErrorCode::INTEREST_TIMEOUT) {
127  auto after = ndn::time::milliseconds(m_jitter(m_rng));
128  NDN_LOG_DEBUG("Schedule sync interest after: " << after);
129  m_scheduledSyncInterestId = m_scheduler.schedule(after, [this] { sendSyncInterest(); });
130  }
131  });
132 
133  NDN_LOG_DEBUG("sendFullSyncInterest, nonce: " << syncInterest.getNonce() <<
134  ", hash: " << std::hash<ndn::Name>{}(syncInterestName));
135 }
136 
137 void
138 FullProducer::onSyncInterest(const ndn::Name& prefixName, const ndn::Interest& interest)
139 {
140  // TODO: answer only segments from store.
141  if (m_segmentPublisher.replyFromStore(interest.getName())) {
142  return;
143  }
144 
145  ndn::Name nameWithoutSyncPrefix = interest.getName().getSubName(prefixName.size());
146  ndn::Name interestName;
147 
148  if (nameWithoutSyncPrefix.size() == 1) {
149  // Get /<prefix>/IBF from /<prefix>/IBF
150  interestName = interest.getName();
151  }
152  else if (nameWithoutSyncPrefix.size() == 3) {
153  // Get /<prefix>/IBF from /<prefix>/IBF/<version>/<segment-no>
154  interestName = interest.getName().getPrefix(-2);
155  }
156  else {
157  return;
158  }
159 
160  ndn::name::Component ibltName = interestName.get(interestName.size() - 1);
161 
162  NDN_LOG_DEBUG("Full sync Interest received, nonce: " << interest.getNonce() <<
163  ", hash: " << std::hash<ndn::Name>{}(interestName));
164 
166  try {
167  iblt.initialize(ibltName);
168  }
169  catch (const std::exception& e) {
170  NDN_LOG_WARN(e.what());
171  return;
172  }
173 
174  auto diff = m_iblt - iblt;
175 
176  std::set<uint32_t> positive;
177  std::set<uint32_t> negative;
178 
179  if (!diff.listEntries(positive, negative)) {
180  NDN_LOG_TRACE("Cannot decode differences, positive: " << positive.size()
181  << " negative: " << negative.size() << " m_threshold: "
182  << m_threshold);
183 
184  // Send all data if greater then threshold, else send positive below as usual
185  // Or send if we can't get neither positive nor negative differences
186  if (positive.size() + negative.size() >= m_threshold ||
187  (positive.size() == 0 && negative.size() == 0)) {
188  detail::State state;
189  for (const auto& content : m_prefixes) {
190  if (content.second != 0) {
191  state.addContent(ndn::Name(content.first).appendNumber(content.second));
192  }
193  }
194 
195  if (!state.getContent().empty()) {
196  sendSyncData(interest.getName(), state.wireEncode());
197  }
198 
199  return;
200  }
201  }
202 
203  detail::State state;
204  for (const auto& hash : positive) {
205  auto nameIt = m_biMap.left.find(hash);
206  if (nameIt != m_biMap.left.end()) {
207  ndn::Name nameWithoutSeq = nameIt->second.getPrefix(-1);
208  // Don't sync up sequence number zero
209  if (m_prefixes[nameWithoutSeq] != 0 &&
210  !isFutureHash(nameWithoutSeq.toUri(), negative)) {
211  state.addContent(nameIt->second);
212  }
213  }
214  }
215 
216  if (!state.getContent().empty()) {
217  NDN_LOG_DEBUG("Sending sync content: " << state);
218  sendSyncData(interestName, state.wireEncode());
219  return;
220  }
221 
222  auto& entry = m_pendingEntries.emplace(interestName, PendingEntryInfo{iblt, {}}).first->second;
223  entry.expirationEvent = m_scheduler.schedule(interest.getInterestLifetime(),
224  [this, interest] {
225  NDN_LOG_TRACE("Erase pending Interest " << interest.getNonce());
226  m_pendingEntries.erase(interest.getName());
227  });
228 }
229 
230 void
231 FullProducer::sendSyncData(const ndn::Name& name, const ndn::Block& block)
232 {
233  NDN_LOG_DEBUG("Checking if data will satisfy our own pending interest");
234 
235  ndn::Name nameWithIblt;
236  m_iblt.appendToName(nameWithIblt);
237 
238  // TODO: Remove appending of hash - serves no purpose to the receiver
239  ndn::Name dataName(ndn::Name(name).appendNumber(std::hash<ndn::Name>{}(nameWithIblt)));
240 
241  auto content = detail::compress(m_contentCompression, block.wire(), block.size());
242 
243  // checking if our own interest got satisfied
244  if (m_outstandingInterestName == name) {
245  NDN_LOG_DEBUG("Satisfied our own pending interest");
246  // remove outstanding interest
247  if (m_fetcher) {
248  NDN_LOG_DEBUG("Removing our pending interest from face (stop fetcher)");
249  m_fetcher->stop();
250  m_outstandingInterestName = ndn::Name("");
251  }
252 
253  NDN_LOG_DEBUG("Sending sync Data");
254 
255  // Send data after removing pending sync interest on face
256  m_segmentPublisher.publish(name, dataName, content, m_syncReplyFreshness);
257 
258  NDN_LOG_TRACE("Renewing sync interest");
259  sendSyncInterest();
260  }
261  else {
262  NDN_LOG_DEBUG("Sending sync Data");
263  m_segmentPublisher.publish(name, dataName, content, m_syncReplyFreshness);
264  }
265 }
266 
267 void
268 FullProducer::onSyncData(const ndn::Interest& interest, const ndn::ConstBufferPtr& bufferPtr)
269 {
270  deletePendingInterests(interest.getName());
271 
272  detail::State state;
273  try {
274  auto decompressed = detail::decompress(m_contentCompression, bufferPtr->data(), bufferPtr->size());
275  state.wireDecode(ndn::Block(std::move(decompressed)));
276  }
277  catch (const std::exception& e) {
278  NDN_LOG_ERROR("Cannot parse received sync Data: " << e.what());
279  return;
280  }
281 
282  NDN_LOG_DEBUG("Sync Data received: " << state);
283 
284  std::vector<MissingDataInfo> updates;
285 
286  for (const auto& content : state) {
287  ndn::Name prefix = content.getPrefix(-1);
288  uint64_t seq = content.get(content.size() - 1).toNumber();
289 
290  if (m_prefixes.find(prefix) == m_prefixes.end() || m_prefixes[prefix] < seq) {
291  updates.push_back({prefix, m_prefixes[prefix] + 1, seq});
292  updateSeqNo(prefix, seq);
293  // We should not call satisfyPendingSyncInterests here because we just
294  // got data and deleted pending interest by calling deletePendingFullSyncInterests
295  // But we might have interests not matching to this interest that might not have deleted
296  // from pending sync interest
297  }
298  }
299 
300  // We just got the data, so send a new sync interest
301  if (!updates.empty()) {
302  m_onUpdate(updates);
303  NDN_LOG_TRACE("Renewing sync interest");
304  sendSyncInterest();
305  }
306  else {
307  NDN_LOG_TRACE("No new update, interest nonce: " << interest.getNonce() <<
308  " , hash: " << std::hash<ndn::Name>{}(interest.getName()));
309  }
310 }
311 
312 void
313 FullProducer::satisfyPendingInterests()
314 {
315  NDN_LOG_DEBUG("Satisfying full sync interest: " << m_pendingEntries.size());
316 
317  for (auto it = m_pendingEntries.begin(); it != m_pendingEntries.end();) {
318  const auto& entry = it->second;
319  auto diff = m_iblt - entry.iblt;
320  std::set<uint32_t> positive;
321  std::set<uint32_t> negative;
322 
323  if (!diff.listEntries(positive, negative)) {
324  NDN_LOG_TRACE("Decode failed for pending interest");
325  if (positive.size() + negative.size() >= m_threshold ||
326  (positive.size() == 0 && negative.size() == 0)) {
327  NDN_LOG_TRACE("pos + neg > threshold or no diff can be found, erase pending interest");
328  it = m_pendingEntries.erase(it);
329  continue;
330  }
331  }
332 
333  detail::State state;
334  for (const auto& hash : positive) {
335  auto nameIt = m_biMap.left.find(hash);
336  if (nameIt != m_biMap.left.end()) {
337  if (m_prefixes[nameIt->second.getPrefix(-1)] != 0) {
338  state.addContent(nameIt->second);
339  }
340  }
341  }
342 
343  if (!state.getContent().empty()) {
344  NDN_LOG_DEBUG("Satisfying sync content: " << state);
345  sendSyncData(it->first, state.wireEncode());
346  it = m_pendingEntries.erase(it);
347  }
348  else {
349  ++it;
350  }
351  }
352 }
353 
354 bool
355 FullProducer::isFutureHash(const ndn::Name& prefix, const std::set<uint32_t>& negative)
356 {
358  ndn::Name(prefix).appendNumber(m_prefixes[prefix] + 1).toUri());
359  return negative.find(nextHash) != negative.end();
360 }
361 
362 void
363 FullProducer::deletePendingInterests(const ndn::Name& interestName)
364 {
365  auto it = m_pendingEntries.find(interestName);
366  if (it != m_pendingEntries.end()) {
367  NDN_LOG_TRACE("Delete pending interest: " << interestName);
368  it = m_pendingEntries.erase(it);
369  }
370 }
371 
372 } // namespace psync
void appendToName(ndn::Name &name) const
Appends self to name.
Definition: iblt.cpp:187
FullProducer(size_t expectedNumEntries, ndn::Face &face, const ndn::Name &syncPrefix, const ndn::Name &userPrefix, const UpdateCallback &onUpdateCallBack, ndn::time::milliseconds syncInterestLifetime=SYNC_INTEREST_LIFTIME, ndn::time::milliseconds syncReplyFreshness=SYNC_REPLY_FRESHNESS, CompressionScheme ibltCompression=CompressionScheme::DEFAULT, CompressionScheme contentCompression=CompressionScheme::DEFAULT)
constructor
void updateSeqNo(const ndn::Name &prefix, uint64_t seq)
Update m_prefixes and IBF with the given prefix and seq.
Invertible Bloom Lookup Table (Invertible Bloom Filter)
Definition: iblt.hpp:81
const ndn::Block & wireEncode() const
Definition: state.cpp:41
Definition: common.hpp:33
NDN_LOG_INIT(psync.Consumer)
HashNameBiMap m_biMap
Full sync logic to synchronize with other nodes where all nodes wants to get all names prefixes synce...
std::shared_ptr< ndn::Buffer > decompress(CompressionScheme scheme, const uint8_t *buffer, size_t bufferSize)
Definition: util.cpp:176
void publishName(const ndn::Name &prefix, ndn::optional< uint64_t > seq=ndn::nullopt)
Publish name to let others know.
void publish(const ndn::Name &interestName, const ndn::Name &dataName, const ndn::Block &block, ndn::time::milliseconds freshness, const ndn::security::SigningInfo &signingInfo=ndn::security::SigningInfo())
Put all the segments in memory.
void addContent(const ndn::Name &prefix)
Definition: state.cpp:34
std::function< void(const std::vector< MissingDataInfo > &)> UpdateCallback
Definition: common.hpp:62
CompressionScheme m_ibltCompression
CompressionScheme
Definition: common.hpp:35
std::shared_ptr< ndn::Buffer > compress(CompressionScheme scheme, const uint8_t *buffer, size_t bufferSize)
Definition: util.cpp:120
bool replyFromStore(const ndn::Name &interestName)
Try to reply from memory, return false if we cannot find the segment.
void initialize(const ndn::name::Component &ibltName)
Populate the hash table using the vector representation of IBLT.
Definition: iblt.cpp:91
CompressionScheme m_contentCompression
const std::vector< ndn::Name > & getContent() const
Definition: state.hpp:48
ndn::random::RandomNumberEngine & m_rng
void onRegisterFailed(const ndn::Name &prefix, const std::string &msg) const
Logs a message if setting an interest filter fails.
ndn::Scheduler m_scheduler
const size_t N_HASHCHECK
ndn::time::milliseconds m_syncReplyFreshness
SegmentPublisher m_segmentPublisher
Base class for PartialProducer and FullProducer.
uint32_t murmurHash3(const void *key, size_t len, uint32_t seed)
Definition: util.cpp:61
std::map< ndn::Name, uint64_t > m_prefixes