Source code

001/**
002 * Licensed to the Apache Software Foundation (ASF) under one or more
003 * contributor license agreements.  See the NOTICE file distributed with
004 * this work for additional information regarding copyright ownership.
005 * The ASF licenses this file to You under the Apache License, Version 2.0
006 * (the "License"); you may not use this file except in compliance with
007 * the License.  You may obtain a copy of the License at
008 *
009 *      http://www.apache.org/licenses/LICENSE-2.0
010 *
011 * Unless required by applicable law or agreed to in writing, software
012 * distributed under the License is distributed on an "AS IS" BASIS,
013 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
014 * See the License for the specific language governing permissions and
015 * limitations under the License.
016 */
017package org.apache.activemq.broker.region.cursors;
018
019import java.util.Iterator;
020import java.util.LinkedList;
021import java.util.ListIterator;
022import java.util.concurrent.CancellationException;
023import java.util.concurrent.Future;
024import java.util.concurrent.TimeUnit;
025import java.util.concurrent.TimeoutException;
026
027import org.apache.activemq.broker.region.Destination;
028import org.apache.activemq.broker.region.MessageReference;
029import org.apache.activemq.broker.region.Subscription;
030import org.apache.activemq.command.Message;
031import org.apache.activemq.command.MessageId;
032import org.apache.activemq.store.MessageRecoveryListener;
033import org.slf4j.Logger;
034import org.slf4j.LoggerFactory;
035
036/**
037 *  Store based cursor
038 *
039 */
040public abstract class AbstractStoreCursor extends AbstractPendingMessageCursor implements MessageRecoveryListener {
041    private static final Logger LOG = LoggerFactory.getLogger(AbstractStoreCursor.class);
042    protected final Destination regionDestination;
043    protected final PendingList batchList;
044    private Iterator<MessageReference> iterator = null;
045    protected boolean batchResetNeeded = false;
046    protected int size;
047    private final LinkedList<MessageId> pendingCachedIds = new LinkedList<>();
048    private static int SYNC_ADD = 0;
049    private static int ASYNC_ADD = 1;
050    final MessageId[] lastCachedIds = new MessageId[2];
051    protected boolean hadSpace = false;
052
053
054
055    protected AbstractStoreCursor(Destination destination) {
056        super((destination != null ? destination.isPrioritizedMessages():false));
057        this.regionDestination=destination;
058        if (this.prioritizedMessages) {
059            this.batchList= new PrioritizedPendingList();
060        } else {
061            this.batchList = new OrderedPendingList();
062        }
063    }
064
065
066    @Override
067    public final synchronized void start() throws Exception{
068        if (!isStarted()) {
069            super.start();
070            resetBatch();
071            resetSize();
072            setCacheEnabled(size==0&&useCache);
073        }
074    }
075
076    protected void resetSize() {
077        this.size = getStoreSize();
078    }
079
080    @Override
081    public void rebase() {
082        resetSize();
083    }
084
085    @Override
086    public final synchronized void stop() throws Exception {
087        resetBatch();
088        super.stop();
089        gc();
090    }
091
092
093    @Override
094    public final boolean recoverMessage(Message message) throws Exception {
095        return recoverMessage(message,false);
096    }
097
098    public synchronized boolean recoverMessage(Message message, boolean cached) throws Exception {
099        boolean recovered = false;
100        message.setRegionDestination(regionDestination);
101        if (recordUniqueId(message.getMessageId())) {
102            if (!cached) {
103                if( message.getMemoryUsage()==null ) {
104                    message.setMemoryUsage(this.getSystemUsage().getMemoryUsage());
105                }
106            }
107            message.incrementReferenceCount();
108            batchList.addMessageLast(message);
109            clearIterator(true);
110            recovered = true;
111        } else if (!cached) {
112            // a duplicate from the store (!cached) - needs to be removed/acked - otherwise it will get re dispatched on restart
113            if (duplicateFromStoreExcepted(message)) {
114                if (LOG.isTraceEnabled()) {
115                    LOG.trace("{} store replayed pending message due to concurrentStoreAndDispatchQueues {} seq: {}", this, message.getMessageId(), message.getMessageId().getFutureOrSequenceLong());
116                }
117            } else {
118                LOG.warn("{} - cursor got duplicate from store {} seq: {}", this, message.getMessageId(), message.getMessageId().getFutureOrSequenceLong());
119                duplicate(message);
120            }
121        } else {
122            LOG.warn("{} - cursor got duplicate send {} seq: {}", this, message.getMessageId(), message.getMessageId().getFutureOrSequenceLong());
123            if (gotToTheStore(message)) {
124                duplicate(message);
125            }
126        }
127        return recovered;
128    }
129
130    protected boolean duplicateFromStoreExcepted(Message message) {
131        // expected for messages pending acks with kahadb.concurrentStoreAndDispatchQueues=true for
132        // which this existing unused flag has been repurposed
133        return message.isRecievedByDFBridge();
134    }
135
136    public static boolean gotToTheStore(Message message) throws Exception {
137        if (message.isRecievedByDFBridge()) {
138            // concurrent store and dispatch - wait to see if the message gets to the store to see
139            // if the index suppressed it (original still present), or whether it was stored and needs to be removed
140            Object possibleFuture = message.getMessageId().getFutureOrSequenceLong();
141            if (possibleFuture instanceof Future) {
142                ((Future) possibleFuture).get();
143            }
144            // need to access again after wait on future
145            Object sequence = message.getMessageId().getFutureOrSequenceLong();
146            return (sequence != null && sequence instanceof Long && Long.compare((Long) sequence, -1l) != 0);
147        }
148        return true;
149    }
150
151    // track for processing outside of store index lock so we can dlq
152    final LinkedList<Message> duplicatesFromStore = new LinkedList<Message>();
153    private void duplicate(Message message) {
154        duplicatesFromStore.add(message);
155    }
156
157    void dealWithDuplicates() {
158        for (Message message : duplicatesFromStore) {
159            regionDestination.duplicateFromStore(message, getSubscription());
160        }
161        duplicatesFromStore.clear();
162    }
163
164    @Override
165    public final synchronized void reset() {
166        if (batchList.isEmpty()) {
167            try {
168                fillBatch();
169            } catch (Exception e) {
170                LOG.error("{} - Failed to fill batch", this, e);
171                throw new RuntimeException(e);
172            }
173        }
174        clearIterator(true);
175        size();
176    }
177
178
179    @Override
180    public synchronized void release() {
181        clearIterator(false);
182    }
183
184    private synchronized void clearIterator(boolean ensureIterator) {
185        boolean haveIterator = this.iterator != null;
186        this.iterator=null;
187        if(haveIterator&&ensureIterator) {
188            ensureIterator();
189        }
190    }
191
192    private synchronized void ensureIterator() {
193        if(this.iterator==null) {
194            this.iterator=this.batchList.iterator();
195        }
196    }
197
198
199    public final void finished() {
200    }
201
202
203    @Override
204    public final synchronized boolean hasNext() {
205        if (batchList.isEmpty()) {
206            try {
207                fillBatch();
208            } catch (Exception e) {
209                LOG.error("{} - Failed to fill batch", this, e);
210                throw new RuntimeException(e);
211            }
212        }
213        ensureIterator();
214        return this.iterator.hasNext();
215    }
216
217
218    @Override
219    public final synchronized MessageReference next() {
220        MessageReference result = null;
221        if (!this.batchList.isEmpty()&&this.iterator.hasNext()) {
222            result = this.iterator.next();
223        }
224        last = result;
225        if (result != null) {
226            result.incrementReferenceCount();
227        }
228        return result;
229    }
230
231    @Override
232    public synchronized boolean tryAddMessageLast(MessageReference node, long wait) throws Exception {
233        boolean disableCache = false;
234        if (hasSpace()) {
235            if (isCacheEnabled()) {
236                if (recoverMessage(node.getMessage(),true)) {
237                    trackLastCached(node);
238                } else {
239                    dealWithDuplicates();
240                    return false;
241                }
242            }
243        } else {
244            disableCache = true;
245        }
246
247        if (disableCache && isCacheEnabled()) {
248            if (LOG.isTraceEnabled()) {
249                LOG.trace("{} - disabling cache on add {} {}", this, node.getMessageId(), node.getMessageId().getFutureOrSequenceLong());
250            }
251            syncWithStore(node.getMessage());
252            setCacheEnabled(false);
253        }
254        size++;
255        return true;
256    }
257
258    @Override
259    public synchronized boolean isCacheEnabled() {
260        return super.isCacheEnabled() || enableCacheNow();
261    }
262
263    protected boolean enableCacheNow() {
264        boolean result = false;
265        if (canEnableCash()) {
266            setCacheEnabled(true);
267            result = true;
268            if (LOG.isTraceEnabled()) {
269                LOG.trace("{} enabling cache on empty store", this);
270            }
271        }
272        return result;
273    }
274
275    protected boolean canEnableCash() {
276        return useCache && size==0 && hasSpace() && isStarted();
277    }
278
279    @Override
280    public boolean canRecoveryNextMessage() {
281        // Should be safe to recovery messages if the overall memory usage if < 90%
282        return parentHasSpace(90);
283    }
284
285    private void syncWithStore(Message currentAdd) throws Exception {
286        pruneLastCached();
287        for (ListIterator<MessageId> it = pendingCachedIds.listIterator(pendingCachedIds.size()); it.hasPrevious(); ) {
288            MessageId lastPending = it.previous();
289            Object futureOrLong = lastPending.getFutureOrSequenceLong();
290            if (futureOrLong instanceof Future) {
291                Future future = (Future) futureOrLong;
292                if (future.isCancelled()) {
293                    continue;
294                }
295                try {
296                    future.get(5, TimeUnit.SECONDS);
297                    setLastCachedId(ASYNC_ADD, lastPending);
298                } catch (CancellationException ok) {
299                    continue;
300                } catch (TimeoutException potentialDeadlock) {
301                    LOG.debug("{} timed out waiting for async add", this, potentialDeadlock);
302                } catch (Exception worstCaseWeReplay) {
303                    LOG.debug("{} exception waiting for async add", this, worstCaseWeReplay);
304                }
305            } else {
306                setLastCachedId(ASYNC_ADD, lastPending);
307            }
308            break;
309        }
310
311        MessageId candidate = lastCachedIds[ASYNC_ADD];
312        if (candidate != null) {
313            // ensure we don't skip current possibly sync add b/c we waited on the future
314            if (!isAsync(currentAdd) && Long.compare(((Long) currentAdd.getMessageId().getFutureOrSequenceLong()), ((Long) lastCachedIds[ASYNC_ADD].getFutureOrSequenceLong())) < 0) {
315                if (LOG.isTraceEnabled()) {
316                    LOG.trace("no set batch from async:" + candidate.getFutureOrSequenceLong() + " >= than current: " + currentAdd.getMessageId().getFutureOrSequenceLong() + ", " + this);
317                }
318                candidate = null;
319            }
320        }
321        if (candidate == null) {
322            candidate = lastCachedIds[SYNC_ADD];
323        }
324        if (candidate != null) {
325            setBatch(candidate);
326        }
327        // cleanup
328        lastCachedIds[SYNC_ADD] = lastCachedIds[ASYNC_ADD] = null;
329        pendingCachedIds.clear();
330    }
331
332    private void trackLastCached(MessageReference node) {
333        if (isAsync(node.getMessage())) {
334            pruneLastCached();
335            pendingCachedIds.add(node.getMessageId());
336        } else {
337            setLastCachedId(SYNC_ADD, node.getMessageId());
338        }
339    }
340
341    private static final boolean isAsync(Message message) {
342        return message.isRecievedByDFBridge() || message.getMessageId().getFutureOrSequenceLong() instanceof Future;
343    }
344
345    private void pruneLastCached() {
346        for (Iterator<MessageId> it = pendingCachedIds.iterator(); it.hasNext(); ) {
347            MessageId candidate = it.next();
348            final Object futureOrLong = candidate.getFutureOrSequenceLong();
349            if (futureOrLong instanceof Future) {
350                Future future = (Future) futureOrLong;
351                if (future.isCancelled()) {
352                    it.remove();
353                } else {
354                    // we don't want to wait for work to complete
355                    break;
356                }
357            } else {
358                // complete
359                setLastCachedId(ASYNC_ADD, candidate);
360
361                // keep lock step with sync adds while order is preserved
362                if (lastCachedIds[SYNC_ADD] != null) {
363                    long next = 1 + (Long)lastCachedIds[SYNC_ADD].getFutureOrSequenceLong();
364                    if (Long.compare((Long)futureOrLong, next) == 0) {
365                        setLastCachedId(SYNC_ADD, candidate);
366                    }
367                }
368                it.remove();
369            }
370        }
371    }
372
373    private void setLastCachedId(final int index, MessageId candidate) {
374        MessageId lastCacheId = lastCachedIds[index];
375        if (lastCacheId == null) {
376            lastCachedIds[index] = candidate;
377        } else {
378            Object lastCacheFutureOrSequenceLong = lastCacheId.getFutureOrSequenceLong();
379            Object candidateOrSequenceLong = candidate.getFutureOrSequenceLong();
380            if (lastCacheFutureOrSequenceLong == null) { // possibly null for topics
381                lastCachedIds[index] = candidate;
382            } else if (candidateOrSequenceLong != null &&
383                    Long.compare(((Long) candidateOrSequenceLong), ((Long) lastCacheFutureOrSequenceLong)) > 0) {
384                lastCachedIds[index] = candidate;
385            } if (LOG.isTraceEnabled()) {
386                LOG.trace("no set last cached[" + index + "] current:" + lastCacheFutureOrSequenceLong + " <= than candidate: " + candidateOrSequenceLong+ ", " + this);
387            }
388        }
389    }
390
391    protected void setBatch(MessageId messageId) throws Exception {
392    }
393
394
395    @Override
396    public synchronized void addMessageFirst(MessageReference node) throws Exception {
397        setCacheEnabled(false);
398        size++;
399    }
400
401
402    @Override
403    public final synchronized void remove() {
404        size--;
405        if (iterator!=null) {
406            iterator.remove();
407        }
408        if (last != null) {
409            last.decrementReferenceCount();
410        }
411    }
412
413
414    @Override
415    public final synchronized void remove(MessageReference node) {
416        if (batchList.remove(node) != null) {
417            size--;
418            setCacheEnabled(false);
419        }
420    }
421
422
423    @Override
424    public final synchronized void clear() {
425        gc();
426    }
427
428
429    @Override
430    public synchronized void gc() {
431        for (MessageReference msg : batchList) {
432            rollback(msg.getMessageId());
433            msg.decrementReferenceCount();
434        }
435        batchList.clear();
436        clearIterator(false);
437        batchResetNeeded = true;
438        setCacheEnabled(false);
439    }
440
441    @Override
442    protected final synchronized void fillBatch() {
443        if (LOG.isTraceEnabled()) {
444            LOG.trace("{} fillBatch", this);
445        }
446        if (batchResetNeeded) {
447            resetSize();
448            setMaxBatchSize(Math.min(regionDestination.getMaxPageSize(), size));
449            resetBatch();
450            this.batchResetNeeded = false;
451        }
452        if (this.batchList.isEmpty() && this.size >0) {
453            try {
454                doFillBatch();
455            } catch (Exception e) {
456                LOG.error("{} - Failed to fill batch", this, e);
457                throw new RuntimeException(e);
458            }
459        }
460    }
461
462
463    @Override
464    public final synchronized boolean isEmpty() {
465        // negative means more messages added to store through queue.send since last reset
466        return size == 0;
467    }
468
469
470    @Override
471    public final synchronized boolean hasMessagesBufferedToDeliver() {
472        return !batchList.isEmpty();
473    }
474
475
476    @Override
477    public final synchronized int size() {
478        if (size < 0) {
479            this.size = getStoreSize();
480        }
481        return size;
482    }
483
484    @Override
485    public final synchronized long messageSize() {
486        return getStoreMessageSize();
487    }
488
489    @Override
490    public String toString() {
491        return super.toString() + ":" + regionDestination.getActiveMQDestination().getPhysicalName() + ",batchResetNeeded=" + batchResetNeeded
492                    + ",size=" + this.size + ",cacheEnabled=" + isCacheEnabled()
493                    + ",maxBatchSize:" + maxBatchSize + ",hasSpace:" + hasSpace() + ",pendingCachedIds.size:" + pendingCachedIds.size()
494                    + ",lastSyncCachedId:" + lastCachedIds[SYNC_ADD] + ",lastSyncCachedId-seq:" + (lastCachedIds[SYNC_ADD] != null ? lastCachedIds[SYNC_ADD].getFutureOrSequenceLong() : "null")
495                    + ",lastAsyncCachedId:" + lastCachedIds[ASYNC_ADD] + ",lastAsyncCachedId-seq:" + (lastCachedIds[ASYNC_ADD] != null ? lastCachedIds[ASYNC_ADD].getFutureOrSequenceLong() : "null");
496    }
497
498    protected abstract void doFillBatch() throws Exception;
499
500    protected abstract void resetBatch();
501
502    protected abstract int getStoreSize();
503
504    protected abstract long getStoreMessageSize();
505
506    protected abstract boolean isStoreEmpty();
507
508    public Subscription getSubscription() {
509        return null;
510    }
511}